pipeline.go 9.86 KB
Newer Older
bryanl's avatar
bryanl committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
// Copyright 2018 The ksonnet authors
//
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package pipeline

import (
	"bytes"
20
	"encoding/json"
bryanl's avatar
bryanl committed
21
	"io"
bryanl's avatar
bryanl committed
22
	"path/filepath"
bryanl's avatar
bryanl committed
23
	"regexp"
bryanl's avatar
bryanl committed
24
	gostrings "strings"
25

26 27
	log "github.com/sirupsen/logrus"

28 29
	"github.com/ksonnet/ksonnet-lib/ksonnet-gen/astext"
	"github.com/ksonnet/ksonnet-lib/ksonnet-gen/printer"
bryanl's avatar
bryanl committed
30
	"github.com/ksonnet/ksonnet/pkg/app"
bryanl's avatar
bryanl committed
31
	"github.com/ksonnet/ksonnet/pkg/component"
32
	"github.com/ksonnet/ksonnet/pkg/env"
bryanl's avatar
bryanl committed
33
	clustermetadata "github.com/ksonnet/ksonnet/pkg/metadata"
34
	"github.com/ksonnet/ksonnet/pkg/params"
bryanl's avatar
bryanl committed
35
	"github.com/ksonnet/ksonnet/pkg/util/jsonnet"
bryanl's avatar
bryanl committed
36 37
	"github.com/ksonnet/ksonnet/pkg/util/k8s"
	"github.com/ksonnet/ksonnet/pkg/util/strings"
bryanl's avatar
bryanl committed
38 39
	"github.com/pkg/errors"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
40
	"k8s.io/apimachinery/pkg/runtime"
bryanl's avatar
bryanl committed
41 42 43 44 45 46 47 48 49 50 51 52 53 54
)

// OverrideManager overrides the component manager interface for a pipeline.
func OverrideManager(c component.Manager) Opt {
	return func(p *Pipeline) {
		p.cm = c
	}
}

// Opt is an option for configuring Pipeline.
type Opt func(p *Pipeline)

// Pipeline is the ks build pipeline.
type Pipeline struct {
55 56 57 58
	app                 app.App
	envName             string
	cm                  component.Manager
	buildObjectsFn      func(*Pipeline, []string) ([]*unstructured.Unstructured, error)
59 60
	evaluateEnvFn       func(a app.App, envName, components, paramsStr string, opts ...jsonnet.VMOpt) (string, error)
	evaluateEnvParamsFn func(a app.App, sourcePath, paramsStr, envName, moduleName string) (string, error)
bryanl's avatar
bryanl committed
61
	stubModuleFn        func(m component.Module) (string, error)
bryanl's avatar
bryanl committed
62 63 64 65
}

// New creates an instance of Pipeline.
func New(ksApp app.App, envName string, opts ...Opt) *Pipeline {
66
	log.Debugf("creating ks pipeline for environment %q", envName)
bryanl's avatar
bryanl committed
67
	p := &Pipeline{
68 69 70 71 72 73
		app:                 ksApp,
		envName:             envName,
		cm:                  component.DefaultManager,
		buildObjectsFn:      buildObjects,
		evaluateEnvFn:       env.Evaluate,
		evaluateEnvParamsFn: params.EvaluateEnv,
bryanl's avatar
bryanl committed
74
		stubModuleFn:        stubModule,
bryanl's avatar
bryanl committed
75 76 77 78 79 80 81 82 83
	}

	for _, opt := range opts {
		opt(p)
	}

	return p
}

84
// Modules returns the modules that belong to this pipeline.
85 86
func (p *Pipeline) Modules() ([]component.Module, error) {
	return p.cm.Modules(p.app, p.envName)
bryanl's avatar
bryanl committed
87 88 89
}

// EnvParameters creates parameters for a namespace given an environment.
bryanl's avatar
bryanl committed
90
func (p *Pipeline) EnvParameters(moduleName string, inherited bool) (string, error) {
91
	module, err := p.cm.Module(p.app, moduleName)
bryanl's avatar
bryanl committed
92
	if err != nil {
93
		return "", errors.Wrapf(err, "load module %s", moduleName)
bryanl's avatar
bryanl committed
94 95
	}

bryanl's avatar
bryanl committed
96
	paramsStr, err := p.moduleParams(module, inherited)
bryanl's avatar
bryanl committed
97
	if err != nil {
bryanl's avatar
bryanl committed
98
		return "", err
bryanl's avatar
bryanl committed
99 100 101 102
	}

	data, err := p.app.EnvironmentParams(p.envName)
	if err != nil {
bryanl's avatar
bryanl committed
103
		return "", errors.Wrapf(err, "retrieve environment params for %s", p.envName)
bryanl's avatar
bryanl committed
104 105 106 107
	}

	envParams := upgradeParams(p.envName, data)

bryanl's avatar
bryanl committed
108 109 110 111
	env, err := p.app.Environment(p.envName)
	if err != nil {
		return "", errors.Wrapf(err, "load environment %s", p.envName)
	}
112

bryanl's avatar
bryanl committed
113
	vm := jsonnet.NewVM()
bryanl's avatar
bryanl committed
114
	vm.AddJPath(
bryanl's avatar
bryanl committed
115
		env.MakePath(p.app.Root()),
bryanl's avatar
bryanl committed
116
		filepath.Join(p.app.Root(), "lib"),
bryanl's avatar
bryanl committed
117 118
		filepath.Join(p.app.Root(), "vendor"),
	)
bryanl's avatar
bryanl committed
119
	vm.ExtCode("__ksonnet/params", paramsStr)
120
	log.Debugf("[Pipeline.EnvParameters] Evaluating: %v", envParams)
bryanl's avatar
bryanl committed
121 122 123
	return vm.EvaluateSnippet("snippet", string(envParams))
}

bryanl's avatar
bryanl committed
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160
func (p *Pipeline) moduleParams(module component.Module, inherited bool) (string, error) {
	if !inherited {
		return stubModule(module)
	}

	paramsStr, err := module.ResolvedParams(p.envName)
	if err != nil {
		return "", errors.Wrapf(err, "resolve params for %s", module.Name())
	}

	return paramsStr, nil
}

func stubModule(module component.Module) (string, error) {
	componentsObject := map[string]interface{}{}

	components, err := module.Components()
	if err != nil {
		return "", errors.Wrap(err, "loading module components")
	}

	for _, c := range components {
		componentsObject[c.Name(true)] = make(map[string]interface{})
	}

	m := map[string]interface{}{
		"components": componentsObject,
	}

	data, err := json.Marshal(&m)
	if err != nil {
		return "", err
	}

	return string(data), nil
}

bryanl's avatar
bryanl committed
161 162
// Components returns the components that belong to this pipeline.
func (p *Pipeline) Components(filter []string) ([]component.Component, error) {
bryanl's avatar
bryanl committed
163
	modules, err := p.Modules()
bryanl's avatar
bryanl committed
164 165 166 167 168
	if err != nil {
		return nil, err
	}

	components := make([]component.Component, 0)
169 170
	for _, m := range modules {
		members, err := p.cm.Components(p.app, m.Name())
bryanl's avatar
bryanl committed
171 172 173 174 175 176 177 178 179 180 181 182 183
		if err != nil {
			return nil, err
		}

		members = filterComponents(filter, members)
		components = append(components, members...)
	}

	return components, nil
}

// Objects converts components into Kubernetes objects.
func (p *Pipeline) Objects(filter []string) ([]*unstructured.Unstructured, error) {
184 185 186 187 188 189 190
	return p.buildObjectsFn(p, filter)
}

func (p *Pipeline) moduleObjects(module component.Module, filter []string) ([]*unstructured.Unstructured, error) {
	doc := &astext.Object{}

	object, componentMap, err := module.Render(p.envName, filter...)
bryanl's avatar
bryanl committed
191
	if err != nil {
192
		return nil, err
bryanl's avatar
bryanl committed
193 194
	}

195 196
	doc.Fields = append(doc.Fields, object.Fields...)

197
	// apply environment parameters
198
	moduleParamData, err := module.ResolvedParams(p.envName)
199
	if err != nil {
200
		return nil, err
bryanl's avatar
bryanl committed
201 202
	}

203
	envParamsPath, err := env.Path(p.app, p.envName, "params.libsonnet")
204 205 206
	if err != nil {
		return nil, err
	}
bryanl's avatar
bryanl committed
207

208
	envParamData, err := p.evaluateEnvParamsFn(p.app, envParamsPath, moduleParamData, p.envName, module.Name())
209
	if err != nil {
210 211 212
		return nil, err
	}

213 214
	var buf bytes.Buffer
	if err = printer.Fprint(&buf, doc); err != nil {
215 216 217
		return nil, err
	}

bryanl's avatar
bryanl committed
218
	// evaluate module with jsonnet.
219
	evaluated, err := p.evaluateEnvFn(p.app, p.envName, buf.String(), envParamData)
220 221 222
	if err != nil {
		return nil, err
	}
bryanl's avatar
bryanl committed
223

224 225 226
	var m map[string]interface{}

	if err = json.Unmarshal([]byte(evaluated), &m); err != nil {
227 228 229 230 231
		return nil, err
	}

	ret := make([]runtime.Object, 0, len(m))

bryanl's avatar
bryanl committed
232 233 234
	for componentName, v := range m {
		if len(filter) != 0 && !strings.InSlice(componentName, filter) {
			continue
bryanl's avatar
bryanl committed
235 236
		}

bryanl's avatar
bryanl committed
237 238 239 240 241 242 243 244
		componentObject, ok := v.(map[string]interface{})
		if !ok {
			return nil, errors.Errorf("component %q is not an object", componentName)
		}

		labelComponents(componentObject, componentName)

		data, err := json.Marshal(componentObject)
bryanl's avatar
bryanl committed
245
		if err != nil {
246
			return nil, err
bryanl's avatar
bryanl committed
247 248
		}

bryanl's avatar
bryanl committed
249
		componentType, ok := componentMap[componentName]
250 251 252
		if !ok {
			// Items in a list won't end up in this map, so assume they are jsonnet.
			componentType = "jsonnet"
bryanl's avatar
bryanl committed
253 254
		}

255
		var patched string
bryanl's avatar
bryanl committed
256

257 258
		switch componentType {
		case "jsonnet":
259
			patched = string(data)
260
		case "yaml":
bryanl's avatar
bryanl committed
261
			patched, err = params.PatchJSON(string(data), envParamData, componentName)
262
			if err != nil {
bryanl's avatar
bryanl committed
263
				return nil, errors.Wrap(err, "patching YAML/JSON component")
264 265
			}
		}
bryanl's avatar
bryanl committed
266

267 268
		uns, _, err := unstructured.UnstructuredJSONScheme.Decode([]byte(patched), nil, nil)
		if err != nil {
bryanl's avatar
bryanl committed
269
			return nil, errors.Wrap(err, "decoding unstructured")
bryanl's avatar
bryanl committed
270
		}
271
		ret = append(ret, uns)
bryanl's avatar
bryanl committed
272 273
	}

274
	return k8s.FlattenToV1(ret)
bryanl's avatar
bryanl committed
275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298
}

// YAML converts components into YAML.
func (p *Pipeline) YAML(filter []string) (io.Reader, error) {
	objects, err := p.Objects(filter)
	if err != nil {
		return nil, err
	}

	var buf bytes.Buffer
	if err := Fprint(&buf, objects, "yaml"); err != nil {
		return nil, errors.Wrap(err, "convert objects to YAML")
	}

	return &buf, nil
}

func filterComponents(filter []string, components []component.Component) []component.Component {
	if len(filter) == 0 {
		return components
	}

	var out []component.Component
	for _, c := range components {
bryanl's avatar
bryanl committed
299
		if strings.InSlice(c.Name(true), filter) {
bryanl's avatar
bryanl committed
300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315
			out = append(out, c)
		}
	}

	return out
}

var (
	reParamSwap = regexp.MustCompile(`(?m)import "\.\.\/\.\.\/components\/params\.libsonnet"`)
)

// upgradeParams replaces relative params imports with an extVar to handle
// multiple component namespaces.
// NOTE: It warns when it makes a change. This serves as a temporary fix until
// ksonnet generates the correct file.
func upgradeParams(envName, in string) string {
316
	if reParamSwap.MatchString(in) {
317
		log.Warnf("rewriting %q environment params to not use relative paths", envName)
318 319 320 321
		return reParamSwap.ReplaceAllLiteralString(in, `std.extVar("__ksonnet/params")`)
	}

	return in
bryanl's avatar
bryanl committed
322 323
}

324 325 326 327 328 329 330 331 332
func buildObjects(p *Pipeline, filter []string) ([]*unstructured.Unstructured, error) {
	modules, err := p.Modules()
	if err != nil {
		return nil, errors.Wrap(err, "get modules")
	}

	var ret []*unstructured.Unstructured

	for _, m := range modules {
333
		log.WithFields(log.Fields{
334 335 336 337 338 339 340 341 342 343 344 345 346
			"action":      "pipeline",
			"module-name": m.Name(),
		}).Debug("building objects")

		objects, err := p.moduleObjects(m, filter)
		if err != nil {
			return nil, err
		}

		ret = append(ret, objects...)
	}

	return ret, nil
bryanl's avatar
bryanl committed
347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386
}

func labelComponents(m map[string]interface{}, name string) {
	if m["apiVersion"] == "v1" && m["kind"] == "List" {
		list, ok := m["items"].([]interface{})
		if !ok {
			return
		}

		for _, item := range list {
			itemMap, ok := item.(map[string]interface{})
			if !ok {
				continue
			}

			labelComponent(itemMap, name)
		}

		return
	}

	labelComponent(m, name)
}

func labelComponent(m map[string]interface{}, name string) {
	metadata, ok := m["metadata"].(map[string]interface{})
	if !ok {
		metadata = make(map[string]interface{})
		m["metadata"] = metadata
	}

	labels, ok := metadata["labels"].(map[string]interface{})
	if !ok {
		labels = make(map[string]interface{})
		metadata["labels"] = labels
	}

	// TODO: this should be owned by module
	name = gostrings.TrimPrefix(name, "/")
	name = gostrings.Replace(name, "/", ".", -1)
387

bryanl's avatar
bryanl committed
388
	labels[clustermetadata.LabelComponent] = name
389
}