From c3fcf41420450df959f44ab24c001e24b0445e89 Mon Sep 17 00:00:00 2001
From: Alex Clemmer <clemmer.alexander@gmail.com>
Date: Wed, 25 Oct 2017 16:51:00 -0700
Subject: [PATCH] Transition away from cgo, and towards go-jsonnet
---
cmd/version.go | 2 +-
prototype/snippet/jsonnet/snippet.go | 81 +++++++++----
template/expander.go | 20 ++--
utils/acquire.go | 9 +-
utils/nativefuncs.go | 164 +++++++++++++++++----------
utils/nativefuncs_test.go | 32 +++---
6 files changed, 199 insertions(+), 109 deletions(-)
diff --git a/cmd/version.go b/cmd/version.go
index eaafbb7f..22da0736 100644
--- a/cmd/version.go
+++ b/cmd/version.go
@@ -18,8 +18,8 @@ package cmd
import (
"fmt"
+ jsonnet "github.com/google/go-jsonnet"
"github.com/spf13/cobra"
- jsonnet "github.com/strickyak/jsonnet_cgo"
"k8s.io/client-go/pkg/version"
)
diff --git a/prototype/snippet/jsonnet/snippet.go b/prototype/snippet/jsonnet/snippet.go
index a4abda6d..8e672d1b 100644
--- a/prototype/snippet/jsonnet/snippet.go
+++ b/prototype/snippet/jsonnet/snippet.go
@@ -72,20 +72,27 @@ func visit(node ast.Node, imports *[]ast.Import) error {
switch n := node.(type) {
case *ast.Import:
// Add parameter-type imports to the list of replacements.
- if strings.HasPrefix(n.File, paramPrefix) {
- param := strings.TrimPrefix(n.File, paramPrefix)
+ if strings.HasPrefix(n.File.Value, paramPrefix) {
+ param := strings.TrimPrefix(n.File.Value, paramPrefix)
if len(param) < 1 {
return errors.New("There must be a parameter following import param://")
}
*imports = append(*imports, *n)
}
case *ast.Apply:
- for _, arg := range n.Arguments {
+ for _, arg := range n.Arguments.Positional {
err := visit(arg, imports)
if err != nil {
return err
}
}
+
+ for _, arg := range n.Arguments.Named {
+ err := visit(arg.Arg, imports)
+ if err != nil {
+ return err
+ }
+ }
return visit(n.Target, imports)
case *ast.ApplyBrace:
err := visit(n.Left, imports)
@@ -101,11 +108,9 @@ func visit(node ast.Node, imports *[]ast.Import) error {
}
}
case *ast.ArrayComp:
- for _, spec := range n.Specs {
- err := visitCompSpec(spec, imports)
- if err != nil {
- return err
- }
+ err := visitCompSpec(n.Spec, imports)
+ if err != nil {
+ return err
}
return visit(n.Body, imports)
case *ast.Assert:
@@ -137,6 +142,13 @@ func visit(node ast.Node, imports *[]ast.Import) error {
case *ast.Error:
return visit(n.Expr, imports)
case *ast.Function:
+ for _, p := range n.Parameters.Optional {
+ err := visit(p.DefaultArg, imports)
+ if err != nil {
+ return err
+ }
+ }
+
return visit(n.Body, imports)
case *ast.Index:
err := visit(n.Target, imports)
@@ -193,22 +205,10 @@ func visit(node ast.Node, imports *[]ast.Import) error {
return err
}
}
- for _, spec := range n.Specs {
- err := visitCompSpec(spec, imports)
- if err != nil {
- return err
- }
- }
- case *ast.ObjectComprehensionSimple:
- err := visit(n.Field, imports)
+ err := visitCompSpec(n.Spec, imports)
if err != nil {
return err
}
- err = visit(n.Value, imports)
- if err != nil {
- return err
- }
- return visit(n.Array, imports)
case *ast.SuperIndex:
return visit(n.Index, imports)
case *ast.InSuper:
@@ -235,11 +235,40 @@ func visit(node ast.Node, imports *[]ast.Import) error {
return nil
}
-func visitCompSpec(node ast.CompSpec, imports *[]ast.Import) error {
+func visitCompSpec(node ast.ForSpec, imports *[]ast.Import) error {
+ if node.Outer != nil {
+ err := visitCompSpec(*node.Outer, imports)
+ if err != nil {
+ return err
+ }
+ }
+
+ for _, ifspec := range node.Conditions {
+ err := visit(ifspec.Expr, imports)
+ if err != nil {
+ return err
+ }
+ }
return visit(node.Expr, imports)
}
func visitObjectField(node ast.ObjectField, imports *[]ast.Import) error {
+ if node.Method != nil {
+ err := visit(node.Method, imports)
+ if err != nil {
+ return err
+ }
+ }
+
+ if node.Params != nil {
+ for _, p := range node.Params.Optional {
+ err := visit(p.DefaultArg, imports)
+ if err != nil {
+ return err
+ }
+ }
+ }
+
err := visit(node.Expr1, imports)
if err != nil {
return err
@@ -260,6 +289,12 @@ func visitDesugaredObjectField(node ast.DesugaredObjectField, imports *[]ast.Imp
}
func visitLocalBind(node ast.LocalBind, imports *[]ast.Import) error {
+ if node.Fun != nil {
+ err := visit(node.Fun, imports)
+ if err != nil {
+ return err
+ }
+ }
return visit(node.Body, imports)
}
@@ -280,7 +315,7 @@ func replace(jsonnet string, imports []ast.Import) string {
})
for _, im := range imports {
- param := paramReplacementPrefix + strings.TrimPrefix(im.File, paramPrefix) + paramReplacementSuffix
+ param := paramReplacementPrefix + strings.TrimPrefix(im.File.Value, paramPrefix) + paramReplacementSuffix
lineStart := im.Loc().Begin.Line
lineEnd := im.Loc().End.Line
diff --git a/template/expander.go b/template/expander.go
index 961a5274..93ea47b6 100644
--- a/template/expander.go
+++ b/template/expander.go
@@ -8,9 +8,9 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+ jsonnet "github.com/google/go-jsonnet"
"github.com/ksonnet/ksonnet/utils"
log "github.com/sirupsen/logrus"
- jsonnet "github.com/strickyak/jsonnet_cgo"
)
type Expander struct {
@@ -31,7 +31,6 @@ func (spec *Expander) Expand(paths []string) ([]*unstructured.Unstructured, erro
if err != nil {
return nil, err
}
- defer vm.Destroy()
res := []*unstructured.Unstructured{}
for _, path := range paths {
@@ -47,18 +46,23 @@ func (spec *Expander) Expand(paths []string) ([]*unstructured.Unstructured, erro
// JsonnetVM constructs a new jsonnet.VM, according to command line
// flags
func (spec *Expander) jsonnetVM() (*jsonnet.VM, error) {
- vm := jsonnet.Make()
+ vm := jsonnet.MakeVM()
+ importer := jsonnet.FileImporter{
+ JPaths: []string{},
+ }
for _, p := range spec.EnvJPath {
log.Debugln("Adding jsonnet search path", p)
- vm.JpathAdd(p)
+ importer.JPaths = append(importer.JPaths, p)
}
for _, p := range spec.FlagJpath {
log.Debugln("Adding jsonnet search path", p)
- vm.JpathAdd(p)
+ importer.JPaths = append(importer.JPaths, p)
}
+ vm.Importer(&importer)
+
for _, extvar := range spec.ExtVars {
kv := strings.SplitN(extvar, "=", 2)
switch len(kv) {
@@ -92,12 +96,12 @@ func (spec *Expander) jsonnetVM() (*jsonnet.VM, error) {
case 1:
v, present := os.LookupEnv(kv[0])
if present {
- vm.TlaVar(kv[0], v)
+ vm.TLAVar(kv[0], v)
} else {
return nil, fmt.Errorf("Missing environment variable: %s", kv[0])
}
case 2:
- vm.TlaVar(kv[0], kv[1])
+ vm.TLAVar(kv[0], kv[1])
}
}
@@ -110,7 +114,7 @@ func (spec *Expander) jsonnetVM() (*jsonnet.VM, error) {
if err != nil {
return nil, err
}
- vm.TlaVar(kv[0], string(v))
+ vm.TLAVar(kv[0], string(v))
}
for _, extcode := range spec.ExtCodes {
diff --git a/utils/acquire.go b/utils/acquire.go
index 7d983fee..dd93e1f7 100644
--- a/utils/acquire.go
+++ b/utils/acquire.go
@@ -24,8 +24,8 @@ import (
"os"
"path/filepath"
+ jsonnet "github.com/google/go-jsonnet"
log "github.com/sirupsen/logrus"
- jsonnet "github.com/strickyak/jsonnet_cgo"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/yaml"
@@ -126,7 +126,12 @@ func jsonWalk(obj interface{}) ([]interface{}, error) {
}
func jsonnetReader(vm *jsonnet.VM, path string) ([]runtime.Object, error) {
- jsonstr, err := vm.EvaluateFile(path)
+ jsonnetBytes, err := ioutil.ReadFile(path)
+ if err != nil {
+ return nil, err
+ }
+
+ jsonstr, err := vm.EvaluateSnippet(path, string(jsonnetBytes))
if err != nil {
return nil, err
}
diff --git a/utils/nativefuncs.go b/utils/nativefuncs.go
index 30fe7cc8..4db4a67c 100644
--- a/utils/nativefuncs.go
+++ b/utils/nativefuncs.go
@@ -24,7 +24,8 @@ import (
goyaml "github.com/ghodss/yaml"
- jsonnet "github.com/strickyak/jsonnet_cgo"
+ jsonnet "github.com/google/go-jsonnet"
+ "github.com/google/go-jsonnet/ast"
"k8s.io/apimachinery/pkg/util/yaml"
)
@@ -48,61 +49,110 @@ func RegisterNativeFuncs(vm *jsonnet.VM, resolver Resolver) {
// "*FromJson" functions will be replaced by regular native
// version when libjsonnet is able to support this.
- vm.NativeCallback("parseJson", []string{"json"}, func(data []byte) (res interface{}, err error) {
- err = json.Unmarshal(data, &res)
- return
- })
-
- vm.NativeCallback("parseYaml", []string{"yaml"}, func(data []byte) ([]interface{}, error) {
- ret := []interface{}{}
- d := yaml.NewYAMLToJSONDecoder(bytes.NewReader(data))
- for {
- var doc interface{}
- if err := d.Decode(&doc); err != nil {
- if err == io.EOF {
- break
+ vm.NativeFunction(
+ &jsonnet.NativeFunction{
+ Name: "parseJson",
+ Params: ast.Identifiers{"json"},
+ Func: func(dataString []interface{}) (res interface{}, err error) {
+ data := []byte(dataString[0].(string))
+ err = json.Unmarshal(data, &res)
+ return
+ },
+ })
+
+ vm.NativeFunction(
+ &jsonnet.NativeFunction{
+ Name: "parseYaml",
+ Params: ast.Identifiers{"yaml"},
+ Func: func(dataString []interface{}) (interface{}, error) {
+ data := []byte(dataString[0].(string))
+ ret := []interface{}{}
+ d := yaml.NewYAMLToJSONDecoder(bytes.NewReader(data))
+ for {
+ var doc interface{}
+ if err := d.Decode(&doc); err != nil {
+ if err == io.EOF {
+ break
+ }
+ return nil, err
+ }
+ ret = append(ret, doc)
}
- return nil, err
- }
- ret = append(ret, doc)
- }
- return ret, nil
- })
-
- vm.NativeCallback("manifestJsonFromJson", []string{"json", "indent"}, func(data []byte, indent int) (string, error) {
- data = bytes.TrimSpace(data)
- buf := bytes.Buffer{}
- if err := json.Indent(&buf, data, "", strings.Repeat(" ", indent)); err != nil {
- return "", err
- }
- buf.WriteString("\n")
- return buf.String(), nil
- })
-
- vm.NativeCallback("manifestYamlFromJson", []string{"json"}, func(data []byte) (string, error) {
- var input interface{}
- if err := json.Unmarshal(data, &input); err != nil {
- return "", err
- }
- output, err := goyaml.Marshal(input)
- return string(output), err
- })
-
- vm.NativeCallback("resolveImage", []string{"image"}, func(image string) (string, error) {
- return resolveImage(resolver, image)
- })
-
- vm.NativeCallback("escapeStringRegex", []string{"str"}, func(s string) (string, error) {
- return regexp.QuoteMeta(s), nil
- })
-
- vm.NativeCallback("regexMatch", []string{"regex", "string"}, regexp.MatchString)
-
- vm.NativeCallback("regexSubst", []string{"regex", "src", "repl"}, func(regex, src, repl string) (string, error) {
- r, err := regexp.Compile(regex)
- if err != nil {
- return "", err
- }
- return r.ReplaceAllString(src, repl), nil
- })
+ return ret, nil
+ },
+ })
+
+ vm.NativeFunction(
+ &jsonnet.NativeFunction{
+ Name: "manifestJsonFromJson",
+ Params: ast.Identifiers{"json", "indent"},
+ Func: func(data []interface{}) (interface{}, error) {
+ indent := int(data[1].(float64))
+ dataBytes := []byte(data[0].(string))
+ dataBytes = bytes.TrimSpace(dataBytes)
+ buf := bytes.Buffer{}
+ if err := json.Indent(&buf, dataBytes, "", strings.Repeat(" ", indent)); err != nil {
+ return "", err
+ }
+ buf.WriteString("\n")
+ return buf.String(), nil
+ },
+ })
+
+ vm.NativeFunction(
+ &jsonnet.NativeFunction{
+ Name: "manifestYamlFromJson",
+ Params: ast.Identifiers{"json"},
+ Func: func(data []interface{}) (interface{}, error) {
+ var input interface{}
+ dataBytes := []byte(data[0].(string))
+ if err := json.Unmarshal(dataBytes, &input); err != nil {
+ return "", err
+ }
+ output, err := goyaml.Marshal(input)
+ return string(output), err
+ },
+ })
+
+ vm.NativeFunction(
+ &jsonnet.NativeFunction{
+ Name: "resolveImage",
+ Params: ast.Identifiers{"image"},
+ Func: func(image []interface{}) (interface{}, error) {
+ return resolveImage(resolver, image[0].(string))
+ },
+ })
+
+ vm.NativeFunction(
+ &jsonnet.NativeFunction{
+ Name: "escapeStringRegex",
+ Params: ast.Identifiers{"str"},
+ Func: func(s []interface{}) (interface{}, error) {
+ return regexp.QuoteMeta(s[0].(string)), nil
+ },
+ })
+
+ vm.NativeFunction(
+ &jsonnet.NativeFunction{
+ Name: "regexMatch",
+ Params: ast.Identifiers{"regex", "string"},
+ Func: func(s []interface{}) (interface{}, error) {
+ return regexp.MatchString(s[0].(string), s[1].(string))
+ },
+ })
+
+ vm.NativeFunction(
+ &jsonnet.NativeFunction{
+ Name: "regexSubst",
+ Params: ast.Identifiers{"regex", "src", "repl"},
+ Func: func(data []interface{}) (interface{}, error) {
+ regex, src, repl := data[0].(string), data[1].(string), data[2].(string)
+
+ r, err := regexp.Compile(regex)
+ if err != nil {
+ return "", err
+ }
+ return r.ReplaceAllString(src, repl), nil
+ },
+ })
}
diff --git a/utils/nativefuncs_test.go b/utils/nativefuncs_test.go
index 8e6d7a97..62485012 100644
--- a/utils/nativefuncs_test.go
+++ b/utils/nativefuncs_test.go
@@ -18,7 +18,7 @@ package utils
import (
"testing"
- jsonnet "github.com/strickyak/jsonnet_cgo"
+ jsonnet "github.com/google/go-jsonnet"
)
// check there is no err, and a == b.
@@ -31,8 +31,7 @@ func check(t *testing.T, err error, actual, expected string) {
}
func TestParseJson(t *testing.T) {
- vm := jsonnet.Make()
- defer vm.Destroy()
+ vm := jsonnet.MakeVM()
RegisterNativeFuncs(vm, NewIdentityResolver())
_, err := vm.EvaluateSnippet("failtest", `std.native("parseJson")("barf{")`)
@@ -41,17 +40,16 @@ func TestParseJson(t *testing.T) {
}
x, err := vm.EvaluateSnippet("test", `std.native("parseJson")("null")`)
- check(t, err, x, "null\n")
+ check(t, err, x, "null")
x, err = vm.EvaluateSnippet("test", `
local a = std.native("parseJson")('{"foo": 3, "bar": 4}');
a.foo + a.bar`)
- check(t, err, x, "7\n")
+ check(t, err, x, "7")
}
func TestParseYaml(t *testing.T) {
- vm := jsonnet.Make()
- defer vm.Destroy()
+ vm := jsonnet.MakeVM()
RegisterNativeFuncs(vm, NewIdentityResolver())
_, err := vm.EvaluateSnippet("failtest", `std.native("parseYaml")("[barf")`)
@@ -60,22 +58,21 @@ func TestParseYaml(t *testing.T) {
}
x, err := vm.EvaluateSnippet("test", `std.native("parseYaml")("")`)
- check(t, err, x, "[ ]\n")
+ check(t, err, x, "[ ]")
x, err = vm.EvaluateSnippet("test", `
local a = std.native("parseYaml")("foo:\n- 3\n- 4\n")[0];
a.foo[0] + a.foo[1]`)
- check(t, err, x, "7\n")
+ check(t, err, x, "7")
x, err = vm.EvaluateSnippet("test", `
local a = std.native("parseYaml")("---\nhello\n---\nworld");
a[0] + a[1]`)
- check(t, err, x, "\"helloworld\"\n")
+ check(t, err, x, "\"helloworld\"")
}
func TestRegexMatch(t *testing.T) {
- vm := jsonnet.Make()
- defer vm.Destroy()
+ vm := jsonnet.MakeVM()
RegisterNativeFuncs(vm, NewIdentityResolver())
_, err := vm.EvaluateSnippet("failtest", `std.native("regexMatch")("[f", "foo")`)
@@ -84,15 +81,14 @@ func TestRegexMatch(t *testing.T) {
}
x, err := vm.EvaluateSnippet("test", `std.native("regexMatch")("foo.*", "seafood")`)
- check(t, err, x, "true\n")
+ check(t, err, x, "true")
x, err = vm.EvaluateSnippet("test", `std.native("regexMatch")("bar.*", "seafood")`)
- check(t, err, x, "false\n")
+ check(t, err, x, "false")
}
func TestRegexSubst(t *testing.T) {
- vm := jsonnet.Make()
- defer vm.Destroy()
+ vm := jsonnet.MakeVM()
RegisterNativeFuncs(vm, NewIdentityResolver())
_, err := vm.EvaluateSnippet("failtest", `std.native("regexSubst")("[f",s "foo", "bar")`)
@@ -101,8 +97,8 @@ func TestRegexSubst(t *testing.T) {
}
x, err := vm.EvaluateSnippet("test", `std.native("regexSubst")("a(x*)b", "-ab-axxb-", "T")`)
- check(t, err, x, "\"-T-T-\"\n")
+ check(t, err, x, "\"-T-T-\"")
x, err = vm.EvaluateSnippet("test", `std.native("regexSubst")("a(x*)b", "-ab-axxb-", "${1}W")`)
- check(t, err, x, "\"-W-xxW-\"\n")
+ check(t, err, x, "\"-W-xxW-\"")
}
--
GitLab