Support env param globals

* Adds env param globals. These can be used to attach metadata labels to
every component in an environment.
* Print more debugging info for jsonnet vm invocations

Fixes #378
Signed-off-by: bryanl <bryanliles@gmail.com>

actions/param_list.go

@@ -68,12 +68,12 @@ func NewParamList(m map[string]interface{}) (*ParamList, error) {
 
 // Run runs the ParamList action.
 func (pl *ParamList) Run() error {
-	ns, err := pl.cm.Module(pl.app, pl.module)
+	module, err := pl.cm.Module(pl.app, pl.module)
 	if err != nil {
 		return errors.Wrap(err, "could not find namespace")
 	}
 
-	params, err := pl.collectParams(ns)
+	params, err := pl.collectParams(module)
 	if err != nil {
 		return err
 	}
@@ -90,9 +90,9 @@ func (pl *ParamList) Run() error {
 	return nil
 }
 
-func (pl *ParamList) collectParams(ns component.Module) ([]component.ModuleParameter, error) {
+func (pl *ParamList) collectParams(module component.Module) ([]component.ModuleParameter, error) {
 	if pl.componentName == "" {
-		return ns.Params(pl.envName)
+		return module.Params(pl.envName)
 	}
 
 	c, err := pl.cm.Component(pl.app, pl.module, pl.componentName)

cmd/root.go

@@ -20,7 +20,6 @@ import (
 	"encoding/json"
 	goflag "flag"
 	"fmt"
-	"io"
 	"os"
 	"path"
 	"path/filepath"
@@ -39,7 +38,6 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
-	"golang.org/x/crypto/ssh/terminal"
 
 	// Register auth plugins
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
@@ -83,7 +81,11 @@ application configuration to remote clusters.
 		out := cmd.OutOrStderr()
 		log.SetOutput(out)
 
-		logFmt := NewLogFormatter(out)
+		logFmt := &log.TextFormatter{
+			DisableTimestamp:       true,
+			DisableLevelTruncation: true,
+			QuoteEmptyFields:       true,
+		}
 		log.SetFormatter(logFmt)
 
 		verbosity, err := flags.GetCount(flagVerbose)
@@ -172,48 +174,6 @@ func logLevel(verbosity int) log.Level {
 	}
 }
 
-type logFormatter struct {
-	escapes  *terminal.EscapeCodes
-	colorise bool
-}
-
-// NewLogFormatter creates a new log.Formatter customised for writer
-func NewLogFormatter(out io.Writer) log.Formatter {
-	var ret = logFormatter{}
-	if f, ok := out.(*os.File); ok {
-		ret.colorise = terminal.IsTerminal(int(f.Fd()))
-		ret.escapes = terminal.NewTerminal(f, "").Escape
-	}
-	return &ret
-}
-
-func (f *logFormatter) levelEsc(level log.Level) []byte {
-	switch level {
-	case log.DebugLevel:
-		return []byte{}
-	case log.WarnLevel:
-		return f.escapes.Yellow
-	case log.ErrorLevel, log.FatalLevel, log.PanicLevel:
-		return f.escapes.Red
-	default:
-		return f.escapes.Blue
-	}
-}
-
-func (f *logFormatter) Format(e *log.Entry) ([]byte, error) {
-	buf := bytes.Buffer{}
-	if f.colorise {
-		buf.Write(f.levelEsc(e.Level))
-		fmt.Fprintf(&buf, "%-5s ", strings.ToUpper(e.Level.String()))
-		buf.Write(f.escapes.Reset)
-	}
-
-	buf.WriteString(strings.TrimSpace(e.Message))
-	buf.WriteString("\n")
-
-	return buf.Bytes(), nil
-}
-
 func newExpander(fs afero.Fs, cmd *cobra.Command) (*template.Expander, error) {
 	flags := cmd.Flags()
 	spec := template.NewExpander(fs)

component/helpers_test.go

+// Copyright 2018 The ksonnet authors
+//
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+
+package component
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/ksonnet/ksonnet/metadata/app/mocks"
+	"github.com/ksonnet/ksonnet/pkg/util/test"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+)
+
+func withAppOsFs(t *testing.T, root string, fn func(*mocks.App, afero.Fs)) {
+	dir, err := ioutil.TempDir("", "")
+	require.NoError(t, err)
+
+	defer os.RemoveAll(dir)
+
+	fs := afero.NewBasePathFs(afero.NewOsFs(), dir)
+
+	test.WithAppFs(t, root, fs, fn)
+}

component/jsonnet.go

@@ -24,9 +24,9 @@ import (
 	"strconv"
 	"strings"
 
-	jsonnet "github.com/google/go-jsonnet"
 	"github.com/ksonnet/ksonnet/metadata/app"
 	"github.com/ksonnet/ksonnet/pkg/params"
+	"github.com/ksonnet/ksonnet/pkg/util/jsonnet"
 	"github.com/ksonnet/ksonnet/pkg/util/k8s"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
@@ -41,6 +41,8 @@ type Jsonnet struct {
 	module     string
 	source     string
 	paramsPath string
+
+	useJsonnetMemoryImporter bool
 }
 
 var _ Component = (*Jsonnet)(nil)
@@ -70,42 +72,42 @@ func (j *Jsonnet) Name(wantsNameSpaced bool) string {
 	return path.Join(j.module, name)
 }
 
-func (j *Jsonnet) vmImporter(envName string) (*jsonnet.MemoryImporter, error) {
-	libPath, err := j.app.LibPath(envName)
-	if err != nil {
-		return nil, err
-	}
-
-	readString := func(path string) (string, error) {
-		filename := filepath.Join(libPath, path)
-		var b []byte
-
-		b, err = afero.ReadFile(j.app.Fs(), filename)
-		if err != nil {
-			return "", err
-		}
-
-		return string(b), nil
-	}
-
-	dataK, err := readString("k.libsonnet")
-	if err != nil {
-		return nil, err
-	}
-	dataK8s, err := readString("k8s.libsonnet")
-	if err != nil {
-		return nil, err
-	}
-
-	importer := &jsonnet.MemoryImporter{
-		Data: map[string]string{
-			"k.libsonnet":   dataK,
-			"k8s.libsonnet": dataK8s,
-		},
-	}
-
-	return importer, nil
-}
+// func (j *Jsonnet) vmImporter(envName string) (*jsonnet.MemoryImporter, error) {
+// 	libPath, err := j.app.LibPath(envName)
+// 	if err != nil {
+// 		return nil, err
+// 	}
+
+// 	readString := func(path string) (string, error) {
+// 		filename := filepath.Join(libPath, path)
+// 		var b []byte
+
+// 		b, err = afero.ReadFile(j.app.Fs(), filename)
+// 		if err != nil {
+// 			return "", err
+// 		}
+
+// 		return string(b), nil
+// 	}
+
+// 	dataK, err := readString("k.libsonnet")
+// 	if err != nil {
+// 		return nil, err
+// 	}
+// 	dataK8s, err := readString("k8s.libsonnet")
+// 	if err != nil {
+// 		return nil, err
+// 	}
+
+// 	importer := &jsonnet.MemoryImporter{
+// 		Data: map[string]string{
+// 			"k.libsonnet":   dataK,
+// 			"k8s.libsonnet": dataK8s,
+// 		},
+// 	}
+
+// 	return importer, nil
+// }
 
 func jsonWalk(obj interface{}) ([]interface{}, error) {
 	switch o := obj.(type) {
@@ -139,15 +141,20 @@ func jsonWalk(obj interface{}) ([]interface{}, error) {
 
 // Objects converts jsonnet to a slice of apimachinery unstructured objects.
 func (j *Jsonnet) Objects(paramsStr, envName string) ([]*unstructured.Unstructured, error) {
-	importer, err := j.vmImporter(envName)
+	libPath, err := j.app.LibPath(envName)
 	if err != nil {
 		return nil, err
 	}
 
-	vm := jsonnet.MakeVM()
-	vm.ErrorFormatter.SetMaxStackTraceSize(40)
-	vm.Importer(importer)
-	log.Debugf("%s convert to jsonnet: setting %q to %q", j.Name(true), "__ksonnet/params", paramsStr)
+	fmt.Println("lib path is", libPath)
+
+	vm := jsonnet.NewVM()
+	if j.useJsonnetMemoryImporter {
+		vm.Fs = j.app.Fs()
+		vm.UseMemoryImporter = true
+	}
+
+	vm.JPaths = []string{libPath}
 	vm.ExtCode("__ksonnet/params", paramsStr)
 
 	snippet, err := afero.ReadFile(j.app.Fs(), j.source)
@@ -155,6 +162,9 @@ func (j *Jsonnet) Objects(paramsStr, envName string) ([]*unstructured.Unstructur
 		return nil, err
 	}
 
+	log.WithFields(log.Fields{
+		"component-name": j.Name(true),
+	}).Debugf("convert component to jsonnet")
 	evaluated, err := vm.EvaluateSnippet(j.source, string(snippet))
 	if err != nil {
 		return nil, err
@@ -312,7 +322,13 @@ func (j *Jsonnet) readParams(envName string) (string, error) {
 
 	envParams := upgradeParams(envName, data)
 
-	vm := jsonnet.MakeVM()
+	env, err := j.app.Environment(envName)
+	if err != nil {
+		return "", err
+	}
+
+	vm := jsonnet.NewVM()
+	vm.JPaths = []string{env.MakePath(j.app.Root())}
 	vm.ExtCode("__ksonnet/params", paramsStr)
 	return vm.EvaluateSnippet("snippet", string(envParams))
 }

component/jsonnet_test.go

@@ -79,8 +79,8 @@ func TestJsonnet_Name(t *testing.T) {
 }
 
 func TestJsonnet_Objects(t *testing.T) {
-	test.WithApp(t, "/app", func(a *mocks.App, fs afero.Fs) {
-		files := []string{"guestbook-ui.jsonnet", "k.libsonnet", "k8s.libsonnet", "params.libsonnet"}
+	withAppOsFs(t, "/app", func(a *mocks.App, fs afero.Fs) {
+		files := []string{"guestbook-ui.jsonnet", "params.libsonnet"}
 		for _, file := range files {
 			test.StageFile(t, fs, "guestbook/"+file, "/app/components/"+file)
 		}
@@ -91,6 +91,7 @@ func TestJsonnet_Objects(t *testing.T) {
 		}
 
 		c := NewJsonnet(a, "", "/app/components/guestbook-ui.jsonnet", "/app/components/params.libsonnet")
+		c.useJsonnetMemoryImporter = true
 
 		paramsStr := testdata(t, "guestbook/params.libsonnet")
 

component/module.go

@@ -28,7 +28,7 @@ import (
 	"github.com/spf13/afero"
 )
 
-func nsErrorMsg(format, module string) string {
+func moduleErrorMsg(format, module string) string {
 	s := fmt.Sprintf("module %q", module)
 	if module == "" {
 		s = "root module"
@@ -66,44 +66,44 @@ func NewModule(ksApp app.App, path string) *FilesystemModule {
 
 // ExtractModuleComponent extracts a module and a component from a path.
 func ExtractModuleComponent(a app.App, path string) (Module, string) {
-	nsPath, component := filepath.Split(path)
-	ns := &FilesystemModule{path: nsPath, app: a}
-	return ns, component
+	modulePath, component := filepath.Split(path)
+	m := &FilesystemModule{path: modulePath, app: a}
+	return m, component
 }
 
 // Name returns the module name.
-func (n *FilesystemModule) Name() string {
-	if n.path == "" {
+func (m *FilesystemModule) Name() string {
+	if m.path == "" {
 		return "/"
 	}
-	return n.path
+	return m.path
 }
 
 // GetModule gets a module by path.
 func GetModule(a app.App, moduleName string) (Module, error) {
 	parts := strings.Split(moduleName, "/")
-	nsDir := filepath.Join(append([]string{a.Root(), componentsRoot}, parts...)...)
+	moduleDir := filepath.Join(append([]string{a.Root(), componentsRoot}, parts...)...)
 
-	exists, err := afero.Exists(a.Fs(), nsDir)
+	exists, err := afero.Exists(a.Fs(), moduleDir)
 	if err != nil {
 		return nil, err
 	}
 
 	if !exists {
-		return nil, errors.New(nsErrorMsg("unable to find %s", moduleName))
+		return nil, errors.New(moduleErrorMsg("unable to find %s", moduleName))
 	}
 
 	return &FilesystemModule{path: moduleName, app: a}, nil
 }
 
 // ParamsPath generates the path to params.libsonnet for a module.
-func (n *FilesystemModule) ParamsPath() string {
-	return filepath.Join(n.Dir(), paramsFile)
+func (m *FilesystemModule) ParamsPath() string {
+	return filepath.Join(m.Dir(), paramsFile)
 }
 
 // SetParam sets params for a module.
-func (n *FilesystemModule) SetParam(path []string, value interface{}) error {
-	paramsData, err := n.readParams()
+func (m *FilesystemModule) SetParam(path []string, value interface{}) error {
+	paramsData, err := m.readParams()
 	if err != nil {
 		return err
 	}
@@ -113,12 +113,12 @@ func (n *FilesystemModule) SetParam(path []string, value interface{}) error {
 		return err
 	}
 
-	return n.writeParams(updated)
+	return m.writeParams(updated)
 }
 
 // DeleteParam deletes params for a module.
-func (n *FilesystemModule) DeleteParam(path []string) error {
-	paramsData, err := n.readParams()
+func (m *FilesystemModule) DeleteParam(path []string) error {
+	paramsData, err := m.readParams()
 	if err != nil {
 		return err
 	}
@@ -128,18 +128,18 @@ func (n *FilesystemModule) DeleteParam(path []string) error {
 		return err
 	}
 
-	return n.writeParams(updated)
+	return m.writeParams(updated)
 }
 
-func (n *FilesystemModule) writeParams(src string) error {
-	return afero.WriteFile(n.app.Fs(), n.ParamsPath(), []byte(src), 0644)
+func (m *FilesystemModule) writeParams(src string) error {
+	return afero.WriteFile(m.app.Fs(), m.ParamsPath(), []byte(src), 0644)
 }
 
-// Dir is the absolute directory for a namespace.
-func (n *FilesystemModule) Dir() string {
-	parts := strings.Split(n.path, "/")
-	path := []string{n.app.Root(), componentsRoot}
-	if len(n.path) != 0 {
+// Dir is the absolute directory for a module.
+func (m *FilesystemModule) Dir() string {
+	parts := strings.Split(m.path, "/")
+	path := []string{m.app.Root(), componentsRoot}
+	if len(m.path) != 0 {
 		path = append(path, parts...)
 	}
 
@@ -156,8 +156,8 @@ type ModuleParameter struct {
 
 // ResolvedParams resolves paramaters for a module. It returns a JSON encoded
 // string of component parameters.
-func (n *FilesystemModule) ResolvedParams() (string, error) {
-	s, err := n.readParams()
+func (m *FilesystemModule) ResolvedParams() (string, error) {
+	s, err := m.readParams()
 	if err != nil {
 		return "", err
 	}
@@ -166,13 +166,13 @@ func (n *FilesystemModule) ResolvedParams() (string, error) {
 }
 
 // Params returns the params for a module.
-func (n *FilesystemModule) Params(envName string) ([]ModuleParameter, error) {
-	components, err := n.Components()
+func (m *FilesystemModule) Params(envName string) ([]ModuleParameter, error) {
+	components, err := m.Components()
 	if err != nil {
 		return nil, err
 	}
 
-	var nsps []ModuleParameter
+	var moduleParameters []ModuleParameter
 	for _, c := range components {
 		params, err := c.Params(envName)
 		if err != nil {
@@ -180,15 +180,15 @@ func (n *FilesystemModule) Params(envName string) ([]ModuleParameter, error) {
 		}
 
 		for _, p := range params {
-			nsps = append(nsps, p)
+			moduleParameters = append(moduleParameters, p)
 		}
 	}
 
-	return nsps, nil
+	return moduleParameters, nil
 }
 
-func (n *FilesystemModule) readParams() (string, error) {
-	b, err := afero.ReadFile(n.app.Fs(), n.ParamsPath())
+func (m *FilesystemModule) readParams() (string, error) {
+	b, err := afero.ReadFile(m.app.Fs(), m.ParamsPath())
 	if err != nil {
 		return "", err
 	}
@@ -206,28 +206,28 @@ func ModulesFromEnv(a app.App, env string) ([]Module, error) {
 	prefix := a.Root() + "/components"
 
 	seen := make(map[string]bool)
-	var namespaces []Module
+	var modules []Module
 	for _, path := range paths {
 		module := strings.TrimPrefix(path, prefix)
 		if _, ok := seen[module]; !ok {
 			seen[module] = true
-			ns, err := GetModule(a, module)
+			m, err := GetModule(a, module)
 			if err != nil {
 				return nil, err
 			}
 
-			namespaces = append(namespaces, ns)
+			modules = append(modules, m)
 		}
 	}
 
-	return namespaces, nil
+	return modules, nil
 }
 
 // Modules returns all component modules
 func Modules(a app.App) ([]Module, error) {
 	componentRoot := filepath.Join(a.Root(), componentsRoot)
 
-	var namespaces []Module
+	var modules []Module
 
 	err := afero.Walk(a.Fs(), componentRoot, func(path string, fi os.FileInfo, err error) error {
 		if err != nil {
@@ -241,10 +241,10 @@ func Modules(a app.App) ([]Module, error) {
 			}
 
 			if ok {
-				nsPath := strings.TrimPrefix(path, componentRoot)
-				nsPath = strings.TrimPrefix(nsPath, string(filepath.Separator))
-				ns := &FilesystemModule{path: nsPath, app: a}
-				namespaces = append(namespaces, ns)
+				modulePath := strings.TrimPrefix(path, componentRoot)
+				modulePath = strings.TrimPrefix(modulePath, string(filepath.Separator))
+				m := &FilesystemModule{path: modulePath, app: a}
+				modules = append(modules, m)
 			}
 		}
 
@@ -255,19 +255,19 @@ func Modules(a app.App) ([]Module, error) {
 		return nil, errors.Wrap(err, "walk component path")
 	}
 
-	sort.Slice(namespaces, func(i, j int) bool {
-		return namespaces[i].Name() < namespaces[j].Name()
+	sort.Slice(modules, func(i, j int) bool {
+		return modules[i].Name() < modules[j].Name()
 	})
 
-	return namespaces, nil
+	return modules, nil
 }
 
 // Components returns the components in a module.
-func (n *FilesystemModule) Components() ([]Component, error) {
-	parts := strings.Split(n.path, "/")
-	nsDir := filepath.Join(append([]string{n.app.Root(), componentsRoot}, parts...)...)
+func (m *FilesystemModule) Components() ([]Component, error) {
+	parts := strings.Split(m.path, "/")
+	moduleDir := filepath.Join(append([]string{m.app.Root(), componentsRoot}, parts...)...)
 
-	fis, err := afero.ReadDir(n.app.Fs(), nsDir)
+	fis, err := afero.ReadDir(m.app.Fs(), moduleDir)
 	if err != nil {
 		return nil, err
 	}
@@ -276,15 +276,15 @@ func (n *FilesystemModule) Components() ([]Component, error) {
 	for _, fi := range fis {
 
 		ext := filepath.Ext(fi.Name())
-		path := filepath.Join(nsDir, fi.Name())
+		path := filepath.Join(moduleDir, fi.Name())
 
 		switch ext {
 		// TODO: these should be constants
 		case ".yaml", ".json":
-			component := NewYAML(n.app, n.Name(), path, n.ParamsPath())
+			component := NewYAML(m.app, m.Name(), path, m.ParamsPath())
 			components = append(components, component)
 		case ".jsonnet":
-			component := NewJsonnet(n.app, n.Name(), path, n.ParamsPath())
+			component := NewJsonnet(m.app, m.Name(), path, m.ParamsPath())
 			components = append(components, component)
 		}
 	}

component/params.go

@@ -18,12 +18,12 @@ package component
 import (
 	"regexp"
 
-	jsonnet "github.com/google/go-jsonnet"
+	"github.com/ksonnet/ksonnet/pkg/util/jsonnet"
 	"github.com/sirupsen/logrus"
 )
 
 func applyGlobals(params string) (string, error) {
-	vm := jsonnet.MakeVM()
+	vm := jsonnet.NewVM()
 
 	vm.ExtCode("params", params)
 	return vm.EvaluateSnippet("snippet", snippetMapGlobal)
@@ -43,7 +43,7 @@ type patchDoc struct {
 }
 
 func patchJSON(jsonObject, patch, patchName string) (string, error) {
-	vm := jsonnet.MakeVM()
+	vm := jsonnet.NewVM()
 	vm.TLACode("target", jsonObject)
 	vm.TLACode("patch", patch)
 	vm.TLAVar("patchName", patchName)

main.go

@@ -35,8 +35,13 @@ func main() {
 	if err := cmd.RootCmd.Execute(); err != nil {
 		// PersistentPreRunE may not have been run for early
 		// errors, like invalid command line flags.
-		logFmt := cmd.NewLogFormatter(log.StandardLogger().Out)
+		logFmt := &log.TextFormatter{
+			DisableTimestamp:       true,
+			DisableLevelTruncation: true,
+			QuoteEmptyFields:       true,
+		}
 		log.SetFormatter(logFmt)
+
 		log.Error(err.Error())
 
 		switch err {

metadata/app/schema.go

@@ -262,6 +262,14 @@ type EnvironmentSpec struct {
 	isOverride bool
 }
 
+// MakePath return the absolute path to the environment directory.
+func (e *EnvironmentSpec) MakePath(rootPath string) string {
+	return filepath.Join(
+		rootPath,
+		EnvironmentDirName,
+		filepath.FromSlash(e.Path))
+}
+
 // IsOverride is true if this EnvironmentSpec is an override.