Skip to content

GitLab

Unverified Commit 03dbafb5 authored by Bryan Liles's avatar Bryan Liles Committed by GitHub
Browse files

Merge pull request #428 from bryanl/bug-jsonnet-param-list 

listing params for yaml components works for more cases
parents daa73e64 00a3cd47
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 28173 additions and 239 deletions
Gopkg.lock
View file @ 03dbafb5
......@@ -18,6 +18,14 @@
revision = "c2a68353555b68de3ee8455a4fd3e890a0ac6d99"
version = "v9.8.1"
[[projects]]
name = "github.com/GeertJohan/go.rice"
packages = [
".",
"embedded"
]
revision = "c02ca9a983da5807ddf7d796784928f5be4afd09"
[[projects]]
name = "github.com/PuerkitoBio/purell"
packages = ["."]
......@@ -47,6 +55,12 @@
revision = "1d903dcb749992f3741d744c0f8376b4bd7eb3e1"
version = "v1.0.7"
[[projects]]
branch = "master"
name = "github.com/daaku/go.zipexe"
packages = ["."]
revision = "a5fe2436ffcb3236e175e5149162b41cd28bd27d"
[[projects]]
name = "github.com/davecgh/go-spew"
packages = ["spew"]
......@@ -250,6 +264,12 @@
packages = ["."]
revision = "59fac5042749a5afb9af70e813da1dd5474f0167"
[[projects]]
branch = "master"
name = "github.com/kardianos/osext"
packages = ["."]
revision = "ae77be60afb1dcacde03767a8c37337fad28ac14"
[[projects]]
name = "github.com/ksonnet/ksonnet-lib"
packages = [
......@@ -675,6 +695,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "e693f795e1fe461827efc6e0d1a0a7e537142eae8440f0e0079d3dc01b4e9887"
inputs-digest = "94ae14fc6f06fe920766e3a94e71635c3071d4afa4a9013b81fc32d23b24d3b5"
solver-name = "gps-cdcl"
solver-version = 1
Gopkg.toml
View file @ 03dbafb5
......@@ -115,3 +115,7 @@
[[constraint]]
name = "k8s.io/client-go"
version = "5.0.0"
[[constraint]]
name = "github.com/GeertJohan/go.rice"
revision = "c02ca9a983da5807ddf7d796784928f5be4afd09"
Makefile
View file @ 03dbafb5
......@@ -65,6 +65,9 @@ vet:
fmt:
$(GOFMT) -s -w $(shell $(GO) list -f '{{.Dir}}' $(GO_PACKAGES))
generate:
$(GO) generate ./...
clean:
$(RM) ./ks ./docs/cli-reference/ks*.md
......
component/component.go
View file @ 03dbafb5
......@@ -20,6 +20,7 @@ import (
"strings"
"github.com/ksonnet/ksonnet/metadata/app"
"github.com/ksonnet/ksonnet/pkg/schema"
"github.com/pkg/errors"
"github.com/spf13/afero"
......@@ -43,8 +44,8 @@ type Summary struct {
}
// GVK converts a summary to a group - version - kind.
func (s *Summary) typeSpec() (*TypeSpec, error) {
return NewTypeSpec(s.APIVersion, s.Kind)
func (s *Summary) typeSpec() (*schema.TypeSpec, error) {
return schema.NewTypeSpec(s.APIVersion, s.Kind)
}
// Component is a ksonnet Component interface.
......
component/jsonnet.go
View file @ 03dbafb5
......@@ -72,43 +72,6 @@ func (j *Jsonnet) Name(wantsNameSpaced bool) string {
return path.Join(j.module, name)
}
// func (j *Jsonnet) vmImporter(envName string) (*jsonnet.MemoryImporter, error) {
// libPath, err := j.app.LibPath(envName)
// if err != nil {
// return nil, err
// }
// readString := func(path string) (string, error) {
// filename := filepath.Join(libPath, path)
// var b []byte
// b, err = afero.ReadFile(j.app.Fs(), filename)
// if err != nil {
// return "", err
// }
// return string(b), nil
// }
// dataK, err := readString("k.libsonnet")
// if err != nil {
// return nil, err
// }
// dataK8s, err := readString("k8s.libsonnet")
// if err != nil {
// return nil, err
// }
// importer := &jsonnet.MemoryImporter{
// Data: map[string]string{
// "k.libsonnet": dataK,
// "k8s.libsonnet": dataK8s,
// },
// }
// return importer, nil
// }
func jsonWalk(obj interface{}) ([]interface{}, error) {
switch o := obj.(type) {
case map[string]interface{}:
......@@ -255,6 +218,8 @@ func (j *Jsonnet) DeleteParam(path []string, options ParamOptions) error {
// Params returns params for a component.
func (j *Jsonnet) Params(envName string) ([]ModuleParameter, error) {
j.log().WithField("env-name", envName).Debug("getting component params")
paramsData, err := j.readParams(envName)
if err != nil {
return nil, err
......@@ -365,3 +330,7 @@ func (j *Jsonnet) readNamespaceParams() (string, error) {
func (j *Jsonnet) writeParams(src string) error {
return afero.WriteFile(j.app.Fs(), j.paramsPath, []byte(src), 0644)
}
func (j *Jsonnet) log() *log.Entry {
return log.WithField("component-name", j.Name(true))
}
component/module.go
View file @ 03dbafb5
......@@ -25,6 +25,7 @@ import (
"github.com/ksonnet/ksonnet/metadata/app"
"github.com/ksonnet/ksonnet/pkg/params"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/afero"
)
......@@ -167,6 +168,8 @@ func (m *FilesystemModule) ResolvedParams() (string, error) {
// Params returns the params for a module.
func (m *FilesystemModule) Params(envName string) ([]ModuleParameter, error) {
m.log().Debug("list module params")
components, err := m.Components()
if err != nil {
return nil, err
......@@ -291,3 +294,7 @@ func (m *FilesystemModule) Components() ([]Component, error) {
return components, nil
}
func (m *FilesystemModule) log() *logrus.Entry {
return logrus.WithField("module-name", m.Name())
}
component/value_extractor_test.go deleted 100644 → 0
View file @ daa73e64
// Copyright 2018 The ksonnet authors
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package component
import (
"testing"
jsonnetutil "github.com/ksonnet/ksonnet/pkg/util/jsonnet"
"github.com/stretchr/testify/require"
)
func TestValueExtractor_Extract(t *testing.T) {
node, err := jsonnetutil.Import("testdata/k8s.libsonnet")
require.NoError(t, err)
props := Properties{
"metadata": map[interface{}]interface{}{
"name": "certificates.certmanager.k8s.io",
"labels": map[interface{}]interface{}{
"app": "cert-manager",
"chart": "cert-manager-0.2.2",
"release": "cert-manager",
"heritage": "Tiller",
},
},
"spec": map[interface{}]interface{}{
"group": "certmanager.k8s.io",
"version": "v1alpha1",
"names": map[interface{}]interface{}{
"kind": "Certificate",
"plural": "certificates",
},
"scope": "Namespaced",
},
}
gvk := GVK{
GroupPath: []string{
"apiextensions.k8s.io",
},
Version: "v1beta1",
Kind: "customResourceDefinition",
}
ve := NewValueExtractor(node)
got, err := ve.Extract(gvk, props)
require.NoError(t, err)
crd := "apiextensions.v1beta1.customResourceDefinition."
expected := map[string]Values{
crd + "mixin.metadata.labels": Values{
Lookup: []string{"metadata", "labels"},
Setter: crd + "mixin.metadata.withLabels",
Value: map[interface{}]interface{}{
"app": "cert-manager",
"chart": "cert-manager-0.2.2",
"release": "cert-manager",
"heritage": "Tiller",
},
},
crd + "mixin.metadata.name": Values{
Lookup: []string{"metadata", "name"},
Setter: crd + "mixin.metadata.withName",
Value: "certificates.certmanager.k8s.io",
},
crd + "mixin.spec.group": Values{
Lookup: []string{"spec", "group"},
Setter: crd + "mixin.spec.withGroup",
Value: "certmanager.k8s.io",
},
crd + "mixin.spec.names.kind": Values{
Lookup: []string{"spec", "names", "kind"},
Setter: crd + "mixin.spec.names.withKind",
Value: "Certificate",
},
crd + "mixin.spec.names.plural": Values{
Lookup: []string{"spec", "names", "plural"},
Setter: crd + "mixin.spec.names.withPlural",
Value: "certificates",
},
crd + "mixin.spec.scope": Values{
Lookup: []string{"spec", "scope"},
Setter: crd + "mixin.spec.withScope",
Value: "Namespaced",
},
crd + "mixin.spec.version": Values{
Lookup: []string{"spec", "version"},
Setter: crd + "mixin.spec.withVersion",
Value: "v1alpha1",
},
}
require.Equal(t, expected, got)
}
component/yaml.go
View file @ 03dbafb5
......@@ -20,7 +20,6 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"path"
"path/filepath"
"regexp"
......@@ -28,15 +27,15 @@ import (
"strconv"
"strings"
jsonnet "github.com/google/go-jsonnet"
"github.com/ksonnet/ksonnet/metadata/app"
"github.com/ksonnet/ksonnet/pkg/params"
"github.com/ksonnet/ksonnet/pkg/schema"
jsonnetutil "github.com/ksonnet/ksonnet/pkg/util/jsonnet"
"github.com/ksonnet/ksonnet/pkg/util/k8s"
utilyaml "github.com/ksonnet/ksonnet/pkg/util/yaml"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/afero"
yaml "gopkg.in/yaml.v2"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
amyaml "k8s.io/apimachinery/pkg/util/yaml"
......@@ -46,51 +45,6 @@ const (
paramsComponentRoot = "components"
)
var (
// ErrEmptyYAML is an empty body error.
ErrEmptyYAML = errors.New("body is empty")
)
// ImportYaml converts a reader containing YAML to a TypeSpec and Properties.
func ImportYaml(r io.Reader) (*TypeSpec, Properties, error) {
b, err := ioutil.ReadAll(r)
if err != nil {
return nil, nil, err
}
if len(b) == 0 {
return nil, nil, ErrEmptyYAML
}
var m map[string]interface{}
if err = yaml.Unmarshal(b, &m); err != nil {
return nil, nil, err
}
props := Properties{}
var kind string
var apiVersion string
for k, v := range m {
switch k {
case "apiVersion":
apiVersion = v.(string)
case "kind":
kind = v.(string)
default:
props[k] = v
}
}
ts, err := NewTypeSpec(apiVersion, kind)
if err != nil {
return nil, nil, err
}
return ts, props, nil
}
// YAML represents a YAML component. Since JSON is a subset of YAML, it can handle JSON as well.
type YAML struct {
app app.App
......@@ -128,19 +82,13 @@ func (y *YAML) Name(wantsNameSpaced bool) string {
// Params returns params for a component.
func (y *YAML) Params(envName string) ([]ModuleParameter, error) {
libPath, err := y.app.LibPath("default")
if err != nil {
return nil, err
}
y.log().WithField("env-name", envName).Debug("getting component params")
k8sPath := filepath.Join(libPath, "k8s.libsonnet")
obj, err := jsonnetutil.ImportFromFs(k8sPath, y.app.Fs())
ve, err := schema.ValueExtractorFactory()
if err != nil {
return nil, err
}
ve := NewValueExtractor(obj)
// find all the params for this component
// keys will look like `component-id`
paramsData, err := y.readParams(envName)
......@@ -165,6 +113,7 @@ func (y *YAML) Params(envName string) ([]ModuleParameter, error) {
var params []ModuleParameter
for componentName, componentValue := range props {
y.log().WithField("prop-name", componentName).Debug("searching for props")
matches := re.FindAllStringSubmatch(componentName, 1)
if len(matches) > 0 {
index := matches[0][1]
......@@ -173,9 +122,9 @@ func (y *YAML) Params(envName string) ([]ModuleParameter, error) {
return nil, err
}
ts, props, err := ImportYaml(readers[i])
ts, props, err := schema.ImportYaml(readers[i])
if err != nil {
if err == ErrEmptyYAML {
if err == schema.ErrEmptyYAML {
continue
}
return nil, err
......@@ -203,7 +152,7 @@ func (y *YAML) Params(envName string) ([]ModuleParameter, error) {
return params, nil
}
func isLeaf(path []string, key string, valueMap map[string]Values) (string, bool) {
func isLeaf(path []string, key string, valueMap map[string]schema.Values) (string, bool) {
childPath := strings.Join(append(path, key), ".")
for _, v := range valueMap {
if strings.Join(v.Lookup, ".") == childPath {
......@@ -214,7 +163,13 @@ func isLeaf(path []string, key string, valueMap map[string]Values) (string, bool
return "", false
}
func (y *YAML) paramValues(componentName, index string, valueMap map[string]Values, m map[string]interface{}, path []string) ([]ModuleParameter, error) {
func (y *YAML) paramValues(componentName, index string, valueMap map[string]schema.Values, m map[string]interface{}, path []string) ([]ModuleParameter, error) {
y.log().WithFields(logrus.Fields{
"prop-name": componentName,
"value-map": fmt.Sprintf("%#v", valueMap),
"m": fmt.Sprintf("%#v", m),
"path": path,
}).Debug("finding param values")
var params []ModuleParameter
for k, v := range m {
......@@ -231,7 +186,6 @@ func (y *YAML) paramValues(componentName, index string, valueMap map[string]Valu
}
params = append(params, p)
}
case map[string]interface{}:
if childPath, exists := isLeaf(path, k, valueMap); exists {
b, err := json.Marshal(&v)
......@@ -273,6 +227,9 @@ func (y *YAML) paramValues(componentName, index string, valueMap map[string]Valu
}
}
if len(params) == 0 {
y.log().Debug("there are no params")
}
return params, nil
}
......@@ -343,6 +300,11 @@ func (y *YAML) readParams(envName string) (string, error) {
return y.readNamespaceParams()
}
libPath, err := y.app.LibPath(envName)
if err != nil {
return "", err
}
ns, err := GetModule(y.app, y.module)
if err != nil {
return "", err
......@@ -360,7 +322,12 @@ func (y *YAML) readParams(envName string) (string, error) {
envParams := upgradeParams(envName, data)
vm := jsonnet.MakeVM()
vm := jsonnetutil.NewVM()
vm.JPaths = []string{
libPath,
filepath.Join(y.app.Root(), "environments", envName),
filepath.Join(y.app.Root(), "vendor"),
}
vm.ExtCode("__ksonnet/params", paramsStr)
return vm.EvaluateSnippet("snippet", string(envParams))
}
......@@ -473,9 +440,9 @@ func (y *YAML) Summarize() ([]Summary, error) {
}
for i, r := range readers {
ts, props, err := ImportYaml(r)
ts, props, err := schema.ImportYaml(r)
if err != nil {
if err == ErrEmptyYAML {
if err == schema.ErrEmptyYAML {
continue
}
return nil, err
......@@ -490,8 +457,8 @@ func (y *YAML) Summarize() ([]Summary, error) {
ComponentName: y.Name(false),
IndexStr: strconv.Itoa(i),
Type: y.ext(),
APIVersion: ts.apiVersion,
Kind: ts.kind,
APIVersion: ts.APIVersion,
Kind: ts.RawKind,
Name: name,
}
summaries = append(summaries, summary)
......@@ -502,7 +469,13 @@ func (y *YAML) Summarize() ([]Summary, error) {
func (y *YAML) ext() string {
return strings.TrimPrefix(filepath.Ext(y.source), ".")
}
func (y *YAML) log() *logrus.Entry {
return logrus.WithFields(logrus.Fields{
"component-name": y.Name(true),
"component-type": "YAML",
})
}
type paramPath struct {
......
component/yaml_test.go
View file @ 03dbafb5
......@@ -31,7 +31,6 @@ func TestYAML_Name(t *testing.T) {
test.StageFile(t, fs, "params-mixed.libsonnet", "/app/components/params.libsonnet")
test.StageFile(t, fs, "deployment.yaml", "/app/components/deployment.yaml")
test.StageFile(t, fs, "k8s.libsonnet", "/app/lib/v1.8.7/k8s.libsonnet")
y := NewYAML(a, "", "/app/components/deployment.yaml", "/app/components/params.libsonnet")
......@@ -66,7 +65,6 @@ func TestYAML_Params(t *testing.T) {
test.StageFile(t, fs, "params-mixed.libsonnet", "/app/components/params.libsonnet")
test.StageFile(t, fs, "deployment.yaml", "/app/components/deployment.yaml")
test.StageFile(t, fs, "k8s.libsonnet", "/app/lib/v1.8.7/k8s.libsonnet")
y := NewYAML(a, "", "/app/components/deployment.yaml", "/app/components/params.libsonnet")
params, err := y.Params("")
......@@ -90,7 +88,6 @@ func TestYAML_Params_literal(t *testing.T) {
test.StageFile(t, fs, "params-mixed.libsonnet", "/params.libsonnet")
test.StageFile(t, fs, "rbac.yaml", "/rbac.yaml")
test.StageFile(t, fs, "k8s.libsonnet", "/app/lib/v1.8.7/k8s.libsonnet")
y := NewYAML(a, "", "/rbac.yaml", "/params.libsonnet")
params, err := y.Params("")
......
e2e/param_test.go
View file @ 03dbafb5
......@@ -18,6 +18,8 @@
package e2e
import (
"path/filepath"
. "github.com/onsi/ginkgo"
)
......@@ -26,7 +28,6 @@ var _ = Describe("ks param", func() {
BeforeEach(func() {
a = e.initApp(nil)
a.generateDeployedService()
})
......@@ -41,6 +42,8 @@ var _ = Describe("ks param", func() {
)
BeforeEach(func() {
a.generateDeployedService()
a.paramSet(component, local, localValue)
a.paramSet(component, env, envValue, "--env", envName)
......@@ -94,31 +97,72 @@ var _ = Describe("ks param", func() {
})
Describe("list", func() {
Context("at the component level", func() {
It("lists the params for a namespace", func() {
o := a.runKs("param", "list")
assertExitStatus(o, 0)
assertOutput("param/list/output.txt", o.stdout)
var (
listOutput *output
listParams = []string{"param", "list", "-v"}
)
JustBeforeEach(func() {
listOutput = a.runKs(listParams...)
})
Describe("at the component level", func() {
Context("with jsonnet component params", func() {
BeforeEach(func() {
a.generateDeployedService()
})
It("should exit with 0", func() {
assertExitStatus(listOutput, 0)
})
It("lists the params for a module", func() {
assertOutput("param/list/output.txt", listOutput.stdout)
})
})
Context("with yaml component params", func() {
BeforeEach(func() {
deployment := filepath.Join(e.wd(), "testdata", "input", "import", "deployment.yaml")
o := a.runKs("import", "-f", deployment)
assertExitStatus(o, 0)
o = a.runKs("param", "set", "deployment", "metadata.labels", `{"hello": "world"}`)
assertExitStatus(o, 0)
})
It("should exit with 0", func() {
assertExitStatus(listOutput, 0)
})
It("should list the YAML params", func() {
assertOutput("param/list/yaml-params.txt", listOutput.stdout)
})
})
})
Context("at the environment level", func() {
It("lists the params for a namespace", func() {
Describe("at the environment level", func() {
BeforeEach(func() {
a.generateDeployedService()
a.paramSet("guestbook-ui", "replicas", "3", "--env", "default")
listParams = []string{"param", "list", "--env", "default"}
})
o := a.paramList()
assertExitStatus(o, 0)
assertOutput("param/list/output.txt", o.stdout)
It("should exit with 0", func() {
assertExitStatus(listOutput, 0)
})
o = a.runKs("param", "list", "--env", "default")
assertExitStatus(o, 0)
assertOutput("param/list/env.txt", o.stdout)
It("lists the params for a module", func() {
assertOutput("param/list/env.txt", listOutput.stdout)
})
})
})
Describe("set", func