Unverified Commit dc80ac49 authored by bryanl's avatar bryanl
Browse files

Support env param globals



* Adds env param globals. These can be used to attach metadata labels to
every component in an environment.
* Print more debugging info for jsonnet vm invocations

Fixes #378
Signed-off-by: default avatarbryanl <bryanliles@gmail.com>
parent f2e95b73
......@@ -68,12 +68,12 @@ func NewParamList(m map[string]interface{}) (*ParamList, error) {
// Run runs the ParamList action.
func (pl *ParamList) Run() error {
ns, err := pl.cm.Module(pl.app, pl.module)
module, err := pl.cm.Module(pl.app, pl.module)
if err != nil {
return errors.Wrap(err, "could not find namespace")
}
params, err := pl.collectParams(ns)
params, err := pl.collectParams(module)
if err != nil {
return err
}
......@@ -90,9 +90,9 @@ func (pl *ParamList) Run() error {
return nil
}
func (pl *ParamList) collectParams(ns component.Module) ([]component.ModuleParameter, error) {
func (pl *ParamList) collectParams(module component.Module) ([]component.ModuleParameter, error) {
if pl.componentName == "" {
return ns.Params(pl.envName)
return module.Params(pl.envName)
}
c, err := pl.cm.Component(pl.app, pl.module, pl.componentName)
......
......@@ -20,7 +20,6 @@ import (
"encoding/json"
goflag "flag"
"fmt"
"io"
"os"
"path"
"path/filepath"
......@@ -39,7 +38,6 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/afero"
"github.com/spf13/cobra"
"golang.org/x/crypto/ssh/terminal"
// Register auth plugins
_ "k8s.io/client-go/plugin/pkg/client/auth"
......@@ -83,7 +81,11 @@ application configuration to remote clusters.
out := cmd.OutOrStderr()
log.SetOutput(out)
logFmt := NewLogFormatter(out)
logFmt := &log.TextFormatter{
DisableTimestamp: true,
DisableLevelTruncation: true,
QuoteEmptyFields: true,
}
log.SetFormatter(logFmt)
verbosity, err := flags.GetCount(flagVerbose)
......@@ -172,48 +174,6 @@ func logLevel(verbosity int) log.Level {
}
}
type logFormatter struct {
escapes *terminal.EscapeCodes
colorise bool
}
// NewLogFormatter creates a new log.Formatter customised for writer
func NewLogFormatter(out io.Writer) log.Formatter {
var ret = logFormatter{}
if f, ok := out.(*os.File); ok {
ret.colorise = terminal.IsTerminal(int(f.Fd()))
ret.escapes = terminal.NewTerminal(f, "").Escape
}
return &ret
}
func (f *logFormatter) levelEsc(level log.Level) []byte {
switch level {
case log.DebugLevel:
return []byte{}
case log.WarnLevel:
return f.escapes.Yellow
case log.ErrorLevel, log.FatalLevel, log.PanicLevel:
return f.escapes.Red
default:
return f.escapes.Blue
}
}
func (f *logFormatter) Format(e *log.Entry) ([]byte, error) {
buf := bytes.Buffer{}
if f.colorise {
buf.Write(f.levelEsc(e.Level))
fmt.Fprintf(&buf, "%-5s ", strings.ToUpper(e.Level.String()))
buf.Write(f.escapes.Reset)
}
buf.WriteString(strings.TrimSpace(e.Message))
buf.WriteString("\n")
return buf.Bytes(), nil
}
func newExpander(fs afero.Fs, cmd *cobra.Command) (*template.Expander, error) {
flags := cmd.Flags()
spec := template.NewExpander(fs)
......
// Copyright 2018 The ksonnet authors
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package component
import (
"io/ioutil"
"os"
"testing"
"github.com/ksonnet/ksonnet/metadata/app/mocks"
"github.com/ksonnet/ksonnet/pkg/util/test"
"github.com/spf13/afero"
"github.com/stretchr/testify/require"
)
func withAppOsFs(t *testing.T, root string, fn func(*mocks.App, afero.Fs)) {
dir, err := ioutil.TempDir("", "")
require.NoError(t, err)
defer os.RemoveAll(dir)
fs := afero.NewBasePathFs(afero.NewOsFs(), dir)
test.WithAppFs(t, root, fs, fn)
}
......@@ -24,9 +24,9 @@ import (
"strconv"
"strings"
jsonnet "github.com/google/go-jsonnet"
"github.com/ksonnet/ksonnet/metadata/app"
"github.com/ksonnet/ksonnet/pkg/params"
"github.com/ksonnet/ksonnet/pkg/util/jsonnet"
"github.com/ksonnet/ksonnet/pkg/util/k8s"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
......@@ -41,6 +41,8 @@ type Jsonnet struct {
module string
source string
paramsPath string
useJsonnetMemoryImporter bool
}
var _ Component = (*Jsonnet)(nil)
......@@ -70,42 +72,42 @@ func (j *Jsonnet) Name(wantsNameSpaced bool) string {
return path.Join(j.module, name)
}
func (j *Jsonnet) vmImporter(envName string) (*jsonnet.MemoryImporter, error) {
libPath, err := j.app.LibPath(envName)
if err != nil {
return nil, err
}
readString := func(path string) (string, error) {
filename := filepath.Join(libPath, path)
var b []byte
b, err = afero.ReadFile(j.app.Fs(), filename)
if err != nil {
return "", err
}
return string(b), nil
}
dataK, err := readString("k.libsonnet")
if err != nil {
return nil, err
}
dataK8s, err := readString("k8s.libsonnet")
if err != nil {
return nil, err
}
importer := &jsonnet.MemoryImporter{
Data: map[string]string{
"k.libsonnet": dataK,
"k8s.libsonnet": dataK8s,
},
}
return importer, nil
}
// func (j *Jsonnet) vmImporter(envName string) (*jsonnet.MemoryImporter, error) {
// libPath, err := j.app.LibPath(envName)
// if err != nil {
// return nil, err
// }
// readString := func(path string) (string, error) {
// filename := filepath.Join(libPath, path)
// var b []byte
// b, err = afero.ReadFile(j.app.Fs(), filename)
// if err != nil {
// return "", err
// }
// return string(b), nil
// }
// dataK, err := readString("k.libsonnet")
// if err != nil {
// return nil, err
// }
// dataK8s, err := readString("k8s.libsonnet")
// if err != nil {
// return nil, err
// }
// importer := &jsonnet.MemoryImporter{
// Data: map[string]string{
// "k.libsonnet": dataK,
// "k8s.libsonnet": dataK8s,
// },
// }
// return importer, nil
// }
func jsonWalk(obj interface{}) ([]interface{}, error) {
switch o := obj.(type) {
......@@ -139,15 +141,20 @@ func jsonWalk(obj interface{}) ([]interface{}, error) {
// Objects converts jsonnet to a slice of apimachinery unstructured objects.
func (j *Jsonnet) Objects(paramsStr, envName string) ([]*unstructured.Unstructured, error) {
importer, err := j.vmImporter(envName)
libPath, err := j.app.LibPath(envName)
if err != nil {
return nil, err
}
vm := jsonnet.MakeVM()
vm.ErrorFormatter.SetMaxStackTraceSize(40)
vm.Importer(importer)
log.Debugf("%s convert to jsonnet: setting %q to %q", j.Name(true), "__ksonnet/params", paramsStr)
fmt.Println("lib path is", libPath)
vm := jsonnet.NewVM()
if j.useJsonnetMemoryImporter {
vm.Fs = j.app.Fs()
vm.UseMemoryImporter = true
}
vm.JPaths = []string{libPath}
vm.ExtCode("__ksonnet/params", paramsStr)
snippet, err := afero.ReadFile(j.app.Fs(), j.source)
......@@ -155,6 +162,9 @@ func (j *Jsonnet) Objects(paramsStr, envName string) ([]*unstructured.Unstructur
return nil, err
}
log.WithFields(log.Fields{
"component-name": j.Name(true),
}).Debugf("convert component to jsonnet")
evaluated, err := vm.EvaluateSnippet(j.source, string(snippet))
if err != nil {
return nil, err
......@@ -312,7 +322,13 @@ func (j *Jsonnet) readParams(envName string) (string, error) {
envParams := upgradeParams(envName, data)
vm := jsonnet.MakeVM()
env, err := j.app.Environment(envName)
if err != nil {
return "", err
}
vm := jsonnet.NewVM()
vm.JPaths = []string{env.MakePath(j.app.Root())}
vm.ExtCode("__ksonnet/params", paramsStr)
return vm.EvaluateSnippet("snippet", string(envParams))
}
......
......@@ -79,8 +79,8 @@ func TestJsonnet_Name(t *testing.T) {
}
func TestJsonnet_Objects(t *testing.T) {
test.WithApp(t, "/app", func(a *mocks.App, fs afero.Fs) {
files := []string{"guestbook-ui.jsonnet", "k.libsonnet", "k8s.libsonnet", "params.libsonnet"}
withAppOsFs(t, "/app", func(a *mocks.App, fs afero.Fs) {
files := []string{"guestbook-ui.jsonnet", "params.libsonnet"}
for _, file := range files {
test.StageFile(t, fs, "guestbook/"+file, "/app/components/"+file)
}
......@@ -91,6 +91,7 @@ func TestJsonnet_Objects(t *testing.T) {
}
c := NewJsonnet(a, "", "/app/components/guestbook-ui.jsonnet", "/app/components/params.libsonnet")
c.useJsonnetMemoryImporter = true
paramsStr := testdata(t, "guestbook/params.libsonnet")
......
......@@ -28,7 +28,7 @@ import (
"github.com/spf13/afero"
)
func nsErrorMsg(format, module string) string {
func moduleErrorMsg(format, module string) string {
s := fmt.Sprintf("module %q", module)
if module == "" {
s = "root module"
......@@ -66,44 +66,44 @@ func NewModule(ksApp app.App, path string) *FilesystemModule {
// ExtractModuleComponent extracts a module and a component from a path.
func ExtractModuleComponent(a app.App, path string) (Module, string) {
nsPath, component := filepath.Split(path)
ns := &FilesystemModule{path: nsPath, app: a}
return ns, component
modulePath, component := filepath.Split(path)
m := &FilesystemModule{path: modulePath, app: a}
return m, component
}
// Name returns the module name.
func (n *FilesystemModule) Name() string {
if n.path == "" {
func (m *FilesystemModule) Name() string {
if m.path == "" {
return "/"
}
return n.path
return m.path
}
// GetModule gets a module by path.
func GetModule(a app.App, moduleName string) (Module, error) {
parts := strings.Split(moduleName, "/")
nsDir := filepath.Join(append([]string{a.Root(), componentsRoot}, parts...)...)
moduleDir := filepath.Join(append([]string{a.Root(), componentsRoot}, parts...)...)
exists, err := afero.Exists(a.Fs(), nsDir)
exists, err := afero.Exists(a.Fs(), moduleDir)
if err != nil {
return nil, err
}
if !exists {
return nil, errors.New(nsErrorMsg("unable to find %s", moduleName))
return nil, errors.New(moduleErrorMsg("unable to find %s", moduleName))
}
return &FilesystemModule{path: moduleName, app: a}, nil
}
// ParamsPath generates the path to params.libsonnet for a module.
func (n *FilesystemModule) ParamsPath() string {
return filepath.Join(n.Dir(), paramsFile)
func (m *FilesystemModule) ParamsPath() string {
return filepath.Join(m.Dir(), paramsFile)
}
// SetParam sets params for a module.
func (n *FilesystemModule) SetParam(path []string, value interface{}) error {
paramsData, err := n.readParams()
func (m *FilesystemModule) SetParam(path []string, value interface{}) error {
paramsData, err := m.readParams()
if err != nil {
return err
}
......@@ -113,12 +113,12 @@ func (n *FilesystemModule) SetParam(path []string, value interface{}) error {
return err
}
return n.writeParams(updated)
return m.writeParams(updated)
}
// DeleteParam deletes params for a module.
func (n *FilesystemModule) DeleteParam(path []string) error {
paramsData, err := n.readParams()
func (m *FilesystemModule) DeleteParam(path []string) error {
paramsData, err := m.readParams()
if err != nil {
return err
}
......@@ -128,18 +128,18 @@ func (n *FilesystemModule) DeleteParam(path []string) error {
return err
}
return n.writeParams(updated)
return m.writeParams(updated)
}
func (n *FilesystemModule) writeParams(src string) error {
return afero.WriteFile(n.app.Fs(), n.ParamsPath(), []byte(src), 0644)
func (m *FilesystemModule) writeParams(src string) error {
return afero.WriteFile(m.app.Fs(), m.ParamsPath(), []byte(src), 0644)
}
// Dir is the absolute directory for a namespace.
func (n *FilesystemModule) Dir() string {
parts := strings.Split(n.path, "/")
path := []string{n.app.Root(), componentsRoot}
if len(n.path) != 0 {
// Dir is the absolute directory for a module.
func (m *FilesystemModule) Dir() string {
parts := strings.Split(m.path, "/")
path := []string{m.app.Root(), componentsRoot}
if len(m.path) != 0 {
path = append(path, parts...)
}
......@@ -156,8 +156,8 @@ type ModuleParameter struct {
// ResolvedParams resolves paramaters for a module. It returns a JSON encoded
// string of component parameters.
func (n *FilesystemModule) ResolvedParams() (string, error) {
s, err := n.readParams()
func (m *FilesystemModule) ResolvedParams() (string, error) {
s, err := m.readParams()
if err != nil {
return "", err
}
......@@ -166,13 +166,13 @@ func (n *FilesystemModule) ResolvedParams() (string, error) {
}
// Params returns the params for a module.
func (n *FilesystemModule) Params(envName string) ([]ModuleParameter, error) {
components, err := n.Components()
func (m *FilesystemModule) Params(envName string) ([]ModuleParameter, error) {
components, err := m.Components()
if err != nil {
return nil, err
}
var nsps []ModuleParameter
var moduleParameters []ModuleParameter
for _, c := range components {
params, err := c.Params(envName)
if err != nil {
......@@ -180,15 +180,15 @@ func (n *FilesystemModule) Params(envName string) ([]ModuleParameter, error) {
}
for _, p := range params {
nsps = append(nsps, p)
moduleParameters = append(moduleParameters, p)
}
}
return nsps, nil
return moduleParameters, nil
}
func (n *FilesystemModule) readParams() (string, error) {
b, err := afero.ReadFile(n.app.Fs(), n.ParamsPath())
func (m *FilesystemModule) readParams() (string, error) {
b, err := afero.ReadFile(m.app.Fs(), m.ParamsPath())
if err != nil {
return "", err
}
......@@ -206,28 +206,28 @@ func ModulesFromEnv(a app.App, env string) ([]Module, error) {
prefix := a.Root() + "/components"
seen := make(map[string]bool)
var namespaces []Module
var modules []Module
for _, path := range paths {
module := strings.TrimPrefix(path, prefix)
if _, ok := seen[module]; !ok {
seen[module] = true
ns, err := GetModule(a, module)
m, err := GetModule(a, module)
if err != nil {
return nil, err
}
namespaces = append(namespaces, ns)
modules = append(modules, m)
}
}
return namespaces, nil
return modules, nil
}
// Modules returns all component modules
func Modules(a app.App) ([]Module, error) {
componentRoot := filepath.Join(a.Root(), componentsRoot)
var namespaces []Module
var modules []Module
err := afero.Walk(a.Fs(), componentRoot, func(path string, fi os.FileInfo, err error) error {
if err != nil {
......@@ -241,10 +241,10 @@ func Modules(a app.App) ([]Module, error) {
}
if ok {
nsPath := strings.TrimPrefix(path, componentRoot)
nsPath = strings.TrimPrefix(nsPath, string(filepath.Separator))
ns := &FilesystemModule{path: nsPath, app: a}
namespaces = append(namespaces, ns)
modulePath := strings.TrimPrefix(path, componentRoot)
modulePath = strings.TrimPrefix(modulePath, string(filepath.Separator))
m := &FilesystemModule{path: modulePath, app: a}
modules = append(modules, m)
}
}
......@@ -255,19 +255,19 @@ func Modules(a app.App) ([]Module, error) {
return nil, errors.Wrap(err, "walk component path")
}
sort.Slice(namespaces, func(i, j int) bool {
return namespaces[i].Name() < namespaces[j].Name()
sort.Slice(modules, func(i, j int) bool {
return modules[i].Name() < modules[j].Name()
})
return namespaces, nil
return modules, nil
}
// Components returns the components in a module.
func (n *FilesystemModule) Components() ([]Component, error) {
parts := strings.Split(n.path, "/")
nsDir := filepath.Join(append([]string{n.app.Root(), componentsRoot}, parts...)...)
func (m *FilesystemModule) Components() ([]Component, error) {
parts := strings.Split(m.path, "/")
moduleDir := filepath.Join(append([]string{m.app.Root(), componentsRoot}, parts...)...)
fis, err := afero.ReadDir(n.app.Fs(), nsDir)
fis, err := afero.ReadDir(m.app.Fs(), moduleDir)
if err != nil {
return nil, err
}
......@@ -276,15 +276,15 @@ func (n *FilesystemModule) Components() ([]Component, error) {
for _, fi := range fis {
ext := filepath.Ext(fi.Name())
path := filepath.Join(nsDir, fi.Name())
path := filepath.Join(moduleDir, fi.Name())
switch ext {
// TODO: these should be constants
case ".yaml", ".json":
component := NewYAML(n.app, n.Name(), path, n.ParamsPath())
component := NewYAML(m.app, m.Name(), path, m.ParamsPath())
components = append(components, component)
case ".jsonnet":
component := NewJsonnet(n.app, n.Name(), path, n.ParamsPath())
component := NewJsonnet(m.app, m.Name(), path, m.ParamsPath())
components = append(components, component)
}
}
......
......@@ -18,12 +18,12 @@ package component
import (
"regexp"
jsonnet "github.com/google/go-jsonnet"
"github.com/ksonnet/ksonnet/pkg/util/jsonnet"
"github.com/sirupsen/logrus"
)
func applyGlobals(params string) (string, error) {
vm := jsonnet.MakeVM()
vm := jsonnet.NewVM()
vm.ExtCode("params", params)
return vm.EvaluateSnippet("snippet", snippetMapGlobal)
......@@ -43,7 +43,7 @@ type patchDoc struct {
}
func patchJSON(jsonObject, patch, patchName string) (string, error) {
vm := jsonnet.MakeVM()
vm := jsonnet.NewVM()
vm.TLACode("target", jsonObject)
vm.TLACode("patch", patch)
vm.TLAVar("patchName", patchName)
......
......@@ -35,8 +35,13 @@ func main() {
if err := cmd.RootCmd.Execute(); err != nil {
// PersistentPreRunE may not have been run for early
// errors, like invalid command line flags.
logFmt := cmd.NewLogFormatter(log.StandardLogger().Out)
logFmt := &log.TextFormatter{
DisableTimestamp: true,
DisableLevelTruncation: true,
QuoteEmptyFields: true,
}
log.SetFormatter(logFmt)
log.Error(err.Error())
switch err {
......
......@@ -262,6 +262,14 @@ type EnvironmentSpec struct {
isOverride bool
}
// MakePath return the absolute path to the environment directory.
func (e *EnvironmentSpec) MakePath(rootPath string) string {
return filepath.Join(
rootPath,
EnvironmentDirName,
filepath.FromSlash(e.Path))
}
// IsOverride is true if this EnvironmentSpec is an override.