Commit f24c1feb authored by Angus Lees's avatar Angus Lees Committed by GitHub
Browse files

Merge pull request #29 from anguslees/nativefuncs

Add native convenience functions to parse JSON and YAML
parents c4e5813a ffc97872
......@@ -14,6 +14,12 @@ matrix:
os: linux
go: '1.8'
allow_failures:
# native functions crash. Unclear if this is a golang bug or
# jsonnet_cgo. Want to fix, but not critical since 1.8 works.
- go: '1.7'
os: osx
addons:
apt:
packages:
......
......@@ -12,9 +12,14 @@ all: kubecfg
kubecfg:
$(GO) build $(GO_FLAGS) .
test:
test: gotest jsonnettest
gotest:
$(GO) test $(GO_FLAGS) $(GO_PACKAGES)
jsonnettest: kubecfg lib/kubecfg_test.jsonnet
./kubecfg -J lib show lib/kubecfg_test.jsonnet
vet:
$(GO) vet $(GO_FLAGS) $(GO_PACKAGES)
......
......@@ -63,6 +63,7 @@ avoid an immediate `Killed: 9`.
- Supports JSON, YAML or jsonnet files (by file suffix).
- Best-effort sorts objects before updating, so that dependencies are
pushed to the server before objects that refer to them.
- Additional jsonnet builtin functions. See `lib/kubecfg.libsonnet`.
## Infrastructure-as-code Philosophy
......
......@@ -91,6 +91,8 @@ func JsonnetVM(cmd *cobra.Command) (*jsonnet.VM, error) {
vm.ExtVar(kv[0], kv[1])
}
utils.RegisterNativeFuncs(vm)
return vm, nil
}
......
{
// parseJson(data): parses the `data` string as a json document, and
// returns the resulting jsonnet object.
parseJson:: std.native("parseJson"),
// parseYaml(data): parse the `data` string as a YAML stream, and
// returns an *array* of the resulting jsonnet objects. A single
// YAML document will still be returned as an array with one
// element.
parseYaml:: std.native("parseYaml"),
}
// Run me with `../kubecfg show kubecfg_test.jsonnet`
local kubecfg = import "kubecfg.libsonnet";
assert kubecfg.parseJson("[3, 4]") == [3, 4];
local x = kubecfg.parseYaml("---
- 3
- 4
---
foo: bar
baz: xyzzy
");
assert x == [[3, 4], {foo: "bar", baz: "xyzzy"}] : "got " + x;
// Kubecfg wants to see something that looks like a k8s object
{
apiVersion: "test",
kind: "Result",
result: "SUCCESS"
}
package utils
import (
"bytes"
"encoding/json"
"io"
jsonnet "github.com/strickyak/jsonnet_cgo"
"k8s.io/client-go/pkg/util/yaml"
)
func RegisterNativeFuncs(vm *jsonnet.VM) {
vm.NativeCallback("parseJson", []string{"json"}, func(data []byte) (res interface{}, err error) {
err = json.Unmarshal(data, &res)
return
})
vm.NativeCallback("parseYaml", []string{"yaml"}, func(data []byte) ([]interface{}, error) {
ret := []interface{}{}
d := yaml.NewYAMLToJSONDecoder(bytes.NewReader(data))
for {
var doc interface{}
if err := d.Decode(&doc); err != nil {
if err == io.EOF {
break
}
return nil, err
}
ret = append(ret, doc)
}
return ret, nil
})
}
package utils
import (
"testing"
jsonnet "github.com/strickyak/jsonnet_cgo"
)
// check there is no err, and a == b.
func check(t *testing.T, err error, actual, expected string) {
if err != nil {
t.Errorf("Expected %q, got error: %q", expected, err.Error())
} else if actual != expected {
t.Errorf("Expected %q, got %q", expected, actual)
}
}
func TestParseJson(t *testing.T) {
vm := jsonnet.Make()
defer vm.Destroy()
RegisterNativeFuncs(vm)
_, err := vm.EvaluateSnippet("failtest", `std.native("parseJson")("barf{")`)
if err == nil {
t.Errorf("parseJson succeeded on invalid json")
}
x, err := vm.EvaluateSnippet("test", `std.native("parseJson")("null")`)
check(t, err, x, "null\n")
x, err = vm.EvaluateSnippet("test", `
local a = std.native("parseJson")('{"foo": 3, "bar": 4}');
a.foo + a.bar`)
check(t, err, x, "7\n")
}
func TestParseYaml(t *testing.T) {
vm := jsonnet.Make()
defer vm.Destroy()
RegisterNativeFuncs(vm)
_, err := vm.EvaluateSnippet("failtest", `std.native("parseYaml")("[barf")`)
if err == nil {
t.Errorf("parseYaml succeeded on invalid yaml")
}
x, err := vm.EvaluateSnippet("test", `std.native("parseYaml")("")`)
check(t, err, x, "[ ]\n")
x, err = vm.EvaluateSnippet("test", `
local a = std.native("parseYaml")("foo:\n- 3\n- 4\n")[0];
a.foo[0] + a.foo[1]`)
check(t, err, x, "7\n")
x, err = vm.EvaluateSnippet("test", `
local a = std.native("parseYaml")("---\nhello\n---\nworld");
a[0] + a[1]`)
check(t, err, x, "\"helloworld\"\n")
}
......@@ -6,17 +6,19 @@
# See "Usage:" a few lines below.
case "$#/$1" in
0/ )
set ../../google/jsonnet/
;;
1/*/jsonnet/ )
: ok
;;
* )
echo >&2 '
Usage:
sh $0 /path/to/jsonnet/
sh $0 ?/path/to/jsonnet/?
This command requires one argument,
the jsonnet repository directory,
ending in /jsonnet/
This command takes one argument, the jsonnet repository directory,
ending in /jsonnet/. The default is ../../google/jsonnet/.
'
exit 13
;;
......
......@@ -2,18 +2,14 @@
#include <stdio.h>
#include <string.h>
#include <libjsonnet.h>
#include "bridge.h"
#include "_cgo_export.h"
char* CallImport_cgo(void *ctx, const char *base, const char *rel, char **found_here, int *success) {
struct JsonnetVm* vm = ctx;
char *path = NULL;
char* result = go_call_import(vm, (char*)base, (char*)rel, &path, success);
if (*success) {
char* found_here_buf = jsonnet_realloc(vm, NULL, strlen(path)+1);
strcpy(found_here_buf, path);
*found_here = found_here_buf;
}
char* buf = jsonnet_realloc(vm, NULL, strlen(result)+1);
strcpy(buf, result);
return buf;
return go_call_import(vm, (char*)base, (char*)rel, found_here, success);
}
struct JsonnetJsonValue* CallNative_cgo(void* ctx, const struct JsonnetJsonValue* const* argv, int* success) {
GoUintptr key = (GoUintptr)ctx;
return go_call_native(key, (struct JsonnetJsonValue**)argv, success);
}
#ifndef LIBJSONNET_BRIDGE_H
#define LIBJSONNET_BRIDGE_H
#include <libjsonnet.h>
typedef JsonnetImportCallback* JsonnetImportCallbackPtr;
struct JsonnetVm* go_get_guts(void* ctx);
char* CallImport_cgo(void *ctx, const char *base, const char *rel, char **found_here, int *success);
char* go_call_import(void* vm, char *base, char *rel, char **path, int *success);
#endif
......@@ -324,7 +324,7 @@ class Desugarer {
};
SuperVars super_vars;
unsigned counter;
unsigned counter = 0;
// Remove +:
for (auto &field : fields) {
......
......@@ -12,34 +12,122 @@ package jsonnet
#include <memory.h>
#include <string.h>
#include <stdio.h>
#include "bridge.h"
#include <stdlib.h>
#include <libjsonnet.h>
char *CallImport_cgo(void *ctx, const char *base, const char *rel, char **found_here, int *success);
struct JsonnetJsonValue *CallNative_cgo(void *ctx, const struct JsonnetJsonValue *const *argv, int *success);
#cgo CXXFLAGS: -std=c++0x -O3
*/
import "C"
import (
"errors"
"fmt"
"reflect"
"runtime"
"sync"
"unsafe"
)
type ImportCallback func(base, rel string) (result string, path string, err error)
type NativeCallback func(args ...*JsonValue) (result *JsonValue, err error)
type nativeFunc struct {
vm *VM
argc int
callback NativeCallback
}
// Global registry of native functions. Cgo pointer rules don't allow
// us to pass go pointers directly (may not be stable), so pass uintptr
// keys into this indirect map instead.
var nativeFuncsMu sync.Mutex
var nativeFuncsIdx uintptr
var nativeFuncs = make(map[uintptr]*nativeFunc)
func registerFunc(vm *VM, arity int, callback NativeCallback) uintptr {
f := nativeFunc{vm: vm, argc: arity, callback: callback}
nativeFuncsMu.Lock()
defer nativeFuncsMu.Unlock()
nativeFuncsIdx++
for nativeFuncs[nativeFuncsIdx] != nil {
nativeFuncsIdx++
}
nativeFuncs[nativeFuncsIdx] = &f
return nativeFuncsIdx
}
func getFunc(key uintptr) *nativeFunc {
nativeFuncsMu.Lock()
defer nativeFuncsMu.Unlock()
return nativeFuncs[key]
}
func unregisterFuncs(vm *VM) {
nativeFuncsMu.Lock()
defer nativeFuncsMu.Unlock()
// This is inefficient if there are many
// simultaneously-existing VMs...
for idx, f := range nativeFuncs {
if f.vm == vm {
delete(nativeFuncs, idx)
}
}
}
type VM struct {
guts *C.struct_JsonnetVm
importCallback ImportCallback
}
//export go_call_native
func go_call_native(key uintptr, argv **C.struct_JsonnetJsonValue, okPtr *C.int) *C.struct_JsonnetJsonValue {
f := getFunc(key)
vm := f.vm
goArgv := make([]*JsonValue, f.argc)
for i := 0; i < f.argc; i++ {
p := unsafe.Pointer(uintptr(unsafe.Pointer(argv)) + unsafe.Sizeof(*argv)*uintptr(i))
argptr := (**C.struct_JsonnetJsonValue)(p)
// NB: argv will be freed by jsonnet after this
// function exits, so don't want (*JsonValue).destroy
// finalizer.
goArgv[i] = &JsonValue{
vm: vm,
guts: *argptr,
}
}
ret, err := f.callback(goArgv...)
if err != nil {
*okPtr = C.int(0)
ret = vm.NewString(err.Error())
} else {
*okPtr = C.int(1)
}
return ret.take()
}
//export go_call_import
func go_call_import(vmPtr unsafe.Pointer, base, rel *C.char, pathPtr **C.char, okPtr *C.int) *C.char {
vm := (*VM)(vmPtr)
result, path, err := vm.importCallback(C.GoString(base), C.GoString(rel))
if err != nil {
*okPtr = C.int(0)
return C.CString(err.Error())
return jsonnetString(vm, err.Error())
}
*pathPtr = C.CString(path)
*pathPtr = jsonnetString(vm, path)
*okPtr = C.int(1)
return C.CString(result)
return jsonnetString(vm, result)
}
// Evaluate a file containing Jsonnet code, return a JSON string.
......@@ -55,10 +143,26 @@ func Make() *VM {
// Complement of Make().
func (vm *VM) Destroy() {
unregisterFuncs(vm)
C.jsonnet_destroy(vm.guts)
vm.guts = nil
}
// jsonnet often wants char* strings that were allocated via
// jsonnet_realloc. This function does that.
func jsonnetString(vm *VM, s string) *C.char {
clen := C.size_t(len(s)) + 1 // num bytes including trailing \0
// TODO: remove additional copy
cstr := C.CString(s)
defer C.free(unsafe.Pointer(cstr))
ret := C.jsonnet_realloc(vm.guts, nil, clen)
C.memcpy(unsafe.Pointer(ret), unsafe.Pointer(cstr), clen)
return ret
}
// Evaluate a file containing Jsonnet code, return a JSON string.
func (vm *VM) EvaluateFile(filename string) (string, error) {
var e C.int
......@@ -79,10 +183,88 @@ func (vm *VM) EvaluateSnippet(filename, snippet string) (string, error) {
return z, nil
}
// Format a file containing Jsonnet code, return a JSON string.
func (vm *VM) FormatFile(filename string) (string, error) {
var e C.int
z := C.GoString(C.jsonnet_fmt_file(vm.guts, C.CString(filename), &e))
if e != 0 {
return "", errors.New(z)
}
return z, nil
}
// Indentation level when reformatting (number of spaces)
func (vm *VM) FormatIndent(n int) {
C.jsonnet_fmt_indent(vm.guts, C.int(n))
}
// Format a string containing Jsonnet code, return a JSON string.
func (vm *VM) FormatSnippet(filename, snippet string) (string, error) {
var e C.int
z := C.GoString(C.jsonnet_fmt_snippet(vm.guts, C.CString(filename), C.CString(snippet), &e))
if e != 0 {
return "", errors.New(z)
}
return z, nil
}
// Override the callback used to locate imports.
func (vm *VM) ImportCallback(f ImportCallback) {
vm.importCallback = f
C.jsonnet_import_callback(vm.guts, C.JsonnetImportCallbackPtr(C.CallImport_cgo), unsafe.Pointer(vm))
C.jsonnet_import_callback(vm.guts, (*C.JsonnetImportCallback)(unsafe.Pointer(C.CallImport_cgo)), unsafe.Pointer(vm))
}
// NativeCallback is a helper around NativeCallbackRaw that uses
// `reflect` to convert argument and result types to/from JsonValue.
// `f` is expected to be a function that takes argument types
// supported by `(*JsonValue).Extract` and returns `(x, error)` where
// `x` is a type supported by `NewJson`.
func (vm *VM) NativeCallback(name string, params []string, f interface{}) {
ty := reflect.TypeOf(f)
if ty.NumIn() != len(params) {
panic("Wrong number of parameters")
}
if ty.NumOut() != 2 {
panic("Wrong number of output parameters")
}
wrapper := func(args ...*JsonValue) (*JsonValue, error) {
in := make([]reflect.Value, len(args))
for i, arg := range args {
value := reflect.ValueOf(arg.Extract())
if vty := value.Type(); !vty.ConvertibleTo(ty.In(i)) {
return nil, fmt.Errorf("parameter %d (type %s) cannot be converted to type %s", i, vty, ty.In(i))
}
in[i] = value.Convert(ty.In(i))
}
out := reflect.ValueOf(f).Call(in)
result := vm.NewJson(out[0].Interface())
var err error
if out[1].IsValid() && !out[1].IsNil() {
err = out[1].Interface().(error)
}
return result, err
}
vm.NativeCallbackRaw(name, params, wrapper)
}
func (vm *VM) NativeCallbackRaw(name string, params []string, f NativeCallback) {
cname := C.CString(name)
defer C.free(unsafe.Pointer(cname))
// jsonnet expects this to be NULL-terminated, so the last
// element is left as nil
cparams := make([]*C.char, len(params)+1)
for i, param := range params {
cparams[i] = C.CString(param)
defer C.free(unsafe.Pointer(cparams[i]))
}
key := registerFunc(vm, len(params), f)
C.jsonnet_native_callback(vm.guts, cname, (*C.JsonnetNativeCallback)(C.CallNative_cgo), unsafe.Pointer(key), (**C.char)(unsafe.Pointer(&cparams[0])))
}
// Bind a Jsonnet external var to the given value.
......@@ -147,3 +329,165 @@ func (vm *VM) JpathAdd(path string) {
* jsonnet_evaluate_file_stream
* jsonnet_evaluate_snippet_stream
*/
// JsonValue represents a jsonnet JSON object.
type JsonValue struct {
vm *VM
guts *C.struct_JsonnetJsonValue
}
func (v *JsonValue) Extract() interface{} {
if x, ok := v.ExtractString(); ok {
return x
}
if x, ok := v.ExtractNumber(); ok {
return x
}
if x, ok := v.ExtractBool(); ok {
return x
}
if ok := v.ExtractNull(); ok {
return nil
}
panic("Unable to extract value")
}
// ExtractString returns the string value and true if the value was a string
func (v *JsonValue) ExtractString() (string, bool) {
cstr := C.jsonnet_json_extract_string(v.vm.guts, v.guts)
if cstr == nil {
return "", false
}
return C.GoString(cstr), true
}
func (v *JsonValue) ExtractNumber() (float64, bool) {
var ret C.double
ok := C.jsonnet_json_extract_number(v.vm.guts, v.guts, &ret)
return float64(ret), ok != 0
}
func (v *JsonValue) ExtractBool() (bool, bool) {
ret := C.jsonnet_json_extract_bool(v.vm.guts, v.guts)
switch ret {
case 0:
return false, true
case 1:
return true, true
case 2:
// Not a bool
return false, false
default:
panic("jsonnet_json_extract_number returned unexpected value")
}
}
// ExtractNull returns true iff the value is null
func (v *JsonValue) ExtractNull() bool {
ret := C.jsonnet_json_extract_null(v.vm.guts, v.guts)
return ret != 0
}
func (vm *VM) newjson(ptr *C.struct_JsonnetJsonValue) *JsonValue {
v := &JsonValue{vm: vm, guts: ptr}
runtime.SetFinalizer(v, (*JsonValue).destroy)
return v
}
func (v *JsonValue) destroy() {
if v.guts == nil {
return
}
C.jsonnet_json_destroy(v.vm.guts, v.guts)
v.guts = nil
runtime.SetFinalizer(v, nil)
}
// Take ownership of the embedded ptr, effectively consuming the JsonValue
func (v *JsonValue) take() *C.struct_JsonnetJsonValue {
ptr := v.guts
if ptr == nil {
panic("taking nil pointer from JsonValue")
}
v.guts = nil
runtime.SetFinalizer(v, nil)
return ptr
}
func (vm *VM) NewJson(value interface{}) *JsonValue {
switch val := value.(type) {
case string:
return vm.NewString(val)
case int:
return vm.NewNumber(float64(val))
case float64:
return vm.NewNumber(val)
case bool:
return vm.NewBool(val)
case nil:
return vm.NewNull()