Commit e2d770a7 authored by Jessica Yuen's avatar Jessica Yuen
Browse files

Add interface for retrieving environment params

The goal of this commit is to add an interface to `metadata.Manager`
that allows retrieving of all environment params:
`getEnvironmentParams(envName)`.

This is done by:

1. Adding functionality to the snippet parser to get all environment
params out of the snippet read from
`environments/<env>/params.libsonnet`

2. Adding functionality to the snippet parser to get all component
params out of the snippet read from `components/params.libsonnet`

3. Having the `metadata.Manager` implementation handle the 'joining' of
params from 1) and 2), and returning that result.
parent 173e0082
......@@ -29,7 +29,7 @@ import (
"github.com/ksonnet/ksonnet-lib/ksonnet-gen/ksonnet"
"github.com/ksonnet/ksonnet-lib/ksonnet-gen/kubespec"
"github.com/ksonnet/ksonnet/metadata/snippet"
param "github.com/ksonnet/ksonnet/metadata/params"
)
const (
......@@ -343,7 +343,37 @@ func (m *manager) SetEnvironment(name string, desired *Environment) error {
return nil
}
func (m *manager) SetEnvironmentParams(env, component string, params map[string]string) error {
func (m *manager) GetEnvironmentParams(name string) (map[string]param.Params, error) {
exists, err := m.environmentExists(name)
if err != nil {
return nil, err
}
if !exists {
return nil, fmt.Errorf("Environment '%s' does not exist", name)
}
// Get the environment specific params
envParamsPath := appendToAbsPath(m.environmentsPath, name, paramsFileName)
envParamsText, err := afero.ReadFile(m.appFS, string(envParamsPath))
if err != nil {
return nil, err
}
envParams, err := param.GetAllEnvironmentParams(string(envParamsText))
if err != nil {
return nil, err
}
// Get all component params
componentParams, err := m.GetAllComponentParams()
if err != nil {
return nil, err
}
// Merge the param sets, replacing the component params if the environment params override
return mergeParamMaps(componentParams, envParams), nil
}
func (m *manager) SetEnvironmentParams(env, component string, params param.Params) error {
exists, err := m.environmentExists(env)
if err != nil {
return err
......@@ -359,7 +389,7 @@ func (m *manager) SetEnvironmentParams(env, component string, params map[string]
return err
}
appended, err := snippet.SetEnvironmentParams(component, string(text), params)
appended, err := param.SetEnvironmentParams(component, string(text), params)
if err != nil {
return err
}
......@@ -443,3 +473,16 @@ func (m *manager) environmentExists(name string) (bool, error) {
return envExists, nil
}
func mergeParamMaps(base, overrides map[string]param.Params) map[string]param.Params {
for component, params := range overrides {
if _, contains := base[component]; !contains {
base[component] = params
} else {
for k, v := range params {
base[component][k] = v
}
}
}
return base
}
......@@ -19,9 +19,11 @@ import (
"encoding/json"
"fmt"
"os"
"reflect"
"strings"
"testing"
param "github.com/ksonnet/ksonnet/metadata/params"
"github.com/spf13/afero"
)
......@@ -224,3 +226,58 @@ params + {
t.Fatalf("Expected to generate params file with data:\n%s\n, got:\n%s", expected, result)
}
}
func TestMergeParamMaps(t *testing.T) {
tests := []struct {
base map[string]param.Params
overrides map[string]param.Params
expected map[string]param.Params
}{
{
map[string]param.Params{
"bar": param.Params{"replicas": "5"},
},
map[string]param.Params{
"foo": param.Params{"name": `"foo"`, "replicas": "1"},
},
map[string]param.Params{
"bar": param.Params{"replicas": "5"},
"foo": param.Params{"name": `"foo"`, "replicas": "1"},
},
},
{
map[string]param.Params{
"bar": param.Params{"replicas": "5"},
},
map[string]param.Params{
"bar": param.Params{"name": `"foo"`},
},
map[string]param.Params{
"bar": param.Params{"name": `"foo"`, "replicas": "5"},
},
},
{
map[string]param.Params{
"bar": param.Params{"name": `"bar"`, "replicas": "5"},
"foo": param.Params{"name": `"foo"`, "replicas": "4"},
"baz": param.Params{"name": `"baz"`, "replicas": "3"},
},
map[string]param.Params{
"foo": param.Params{"replicas": "1"},
"baz": param.Params{"name": `"foobaz"`},
},
map[string]param.Params{
"bar": param.Params{"name": `"bar"`, "replicas": "5"},
"foo": param.Params{"name": `"foo"`, "replicas": "1"},
"baz": param.Params{"name": `"foobaz"`, "replicas": "3"},
},
},
}
for _, s := range tests {
result := mergeParamMaps(s.base, s.overrides)
if !reflect.DeepEqual(s.expected, result) {
t.Errorf("Wrong merge\n expected:\n%v\n got:\n%v", s.expected, result)
}
}
}
......@@ -20,6 +20,7 @@ import (
"regexp"
"strings"
param "github.com/ksonnet/ksonnet/metadata/params"
"github.com/ksonnet/ksonnet/prototype"
"github.com/spf13/afero"
)
......@@ -42,16 +43,22 @@ type AbsPaths []string
type Manager interface {
Root() AbsPath
ComponentPaths() (AbsPaths, error)
CreateComponent(name string, text string, params map[string]string, templateType prototype.TemplateType) error
CreateComponent(name string, text string, params param.Params, templateType prototype.TemplateType) error
LibPaths(envName string) (libPath, envLibPath, envComponentPath, envParamsPath AbsPath)
SetComponentParams(component string, params map[string]string) error
GetComponentParams(name string) (map[string]string, error)
SetComponentParams(component string, params param.Params) error
GetComponentParams(name string) (param.Params, error)
GetAllComponentParams() (map[string]param.Params, error)
CreateEnvironment(name, uri, namespace string, spec ClusterSpec) error
DeleteEnvironment(name string) error
GetEnvironments() ([]*Environment, error)
GetEnvironment(name string) (*Environment, error)
SetEnvironment(name string, desired *Environment) error
SetEnvironmentParams(env, component string, params map[string]string) error
// GetEnvironmentParams will take the name of an environment and return a
// mapping of parameters of the form:
// componentName => {param key => param val}
// i.e.: "nginx" => {"replicas" => 1, "name": "nginx"}
GetEnvironmentParams(name string) (map[string]param.Params, error)
SetEnvironmentParams(env, component string, params param.Params) error
//
// TODO: Fill in methods as we need them.
......@@ -59,7 +66,6 @@ type Manager interface {
// GetPrototype(id string) Protoype
// SearchPrototypes(query string) []Protoype
// VendorLibrary(uri, version string) error
// GetEnvironmentParams(env string) (map[string]map[string]string, error)
}
// Find will recursively search the current directory and its parents for a
......
......@@ -22,7 +22,7 @@ import (
"path/filepath"
"strings"
"github.com/ksonnet/ksonnet/metadata/snippet"
param "github.com/ksonnet/ksonnet/metadata/params"
"github.com/ksonnet/ksonnet/prototype"
log "github.com/sirupsen/logrus"
"github.com/spf13/afero"
......@@ -155,7 +155,7 @@ func (m *manager) ComponentPaths() (AbsPaths, error) {
return paths, nil
}
func (m *manager) CreateComponent(name string, text string, params map[string]string, templateType prototype.TemplateType) error {
func (m *manager) CreateComponent(name string, text string, params param.Params, templateType prototype.TemplateType) error {
if !isValidName(name) || strings.Contains(name, "/") {
return fmt.Errorf("Component name '%s' is not valid; must not contain punctuation, spaces, or begin or end with a slash", name)
}
......@@ -194,22 +194,31 @@ func (m *manager) LibPaths(envName string) (libPath, envLibPath, envComponentPat
appendToAbsPath(envPath, path.Base(envName)+".jsonnet"), appendToAbsPath(envPath, componentParamsFile)
}
func (m *manager) GetComponentParams(component string) (map[string]string, error) {
func (m *manager) GetComponentParams(component string) (param.Params, error) {
text, err := afero.ReadFile(m.appFS, string(m.componentParamsPath))
if err != nil {
return nil, err
}
return snippet.GetComponentParams(component, string(text))
return param.GetComponentParams(component, string(text))
}
func (m *manager) SetComponentParams(component string, params map[string]string) error {
func (m *manager) GetAllComponentParams() (map[string]param.Params, error) {
text, err := afero.ReadFile(m.appFS, string(m.componentParamsPath))
if err != nil {
return nil, err
}
return param.GetAllComponentParams(string(text))
}
func (m *manager) SetComponentParams(component string, params param.Params) error {
text, err := afero.ReadFile(m.appFS, string(m.componentParamsPath))
if err != nil {
return err
}
jsonnet, err := snippet.SetComponentParams(component, string(text), params)
jsonnet, err := param.SetComponentParams(component, string(text), params)
if err != nil {
return err
}
......@@ -263,13 +272,13 @@ func (m *manager) createAppDirTree() error {
return nil
}
func (m *manager) writeComponentParams(componentName string, params map[string]string) error {
func (m *manager) writeComponentParams(componentName string, params param.Params) error {
text, err := afero.ReadFile(m.appFS, string(m.componentParamsPath))
if err != nil {
return err
}
appended, err := snippet.AppendComponent(componentName, string(text), params)
appended, err := param.AppendComponent(componentName, string(text), params)
if err != nil {
return err
}
......
......@@ -13,7 +13,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package snippet
package params
type Params map[string]string
// AppendComponent takes the following params
//
......@@ -22,7 +24,7 @@ package snippet
// params: the parameters for the new component.
//
// and returns the jsonnet snippet with the appended component and parameters.
func AppendComponent(component, snippet string, params map[string]string) (string, error) {
func AppendComponent(component, snippet string, params Params) (string, error) {
return appendComponent(component, snippet, params)
}
......@@ -34,11 +36,20 @@ func AppendComponent(component, snippet string, params map[string]string) (strin
// and returns a map of key-value param pairs corresponding to that component.
//
// An error will be returned if the component is not found in the snippet.
func GetComponentParams(component, snippet string) (map[string]string, error) {
func GetComponentParams(component, snippet string) (Params, error) {
params, _, err := getComponentParams(component, snippet)
return params, err
}
// GetAllComponentParams takes
//
// snippet: the jsonnet snippet containing the component params.
//
// and returns a map of key-value param pairs for each component identified.
func GetAllComponentParams(snippet string) (map[string]Params, error) {
return getAllComponentParams(snippet)
}
// SetComponentParams takes
//
// component: the name of the new component to be modified.
......@@ -46,10 +57,21 @@ func GetComponentParams(component, snippet string) (map[string]string, error) {
// params: the parameters to be set for 'component'.
//
// and returns the jsonnet snippet with the modified set of component parameters.
func SetComponentParams(component, snippet string, params map[string]string) (string, error) {
func SetComponentParams(component, snippet string, params Params) (string, error) {
return setComponentParams(component, snippet, params)
}
// GetAllEnvironmentParams takes
//
// snippet: the jsonnet snippet containing the environment params. This is
// expected to be non-expanded schema; i.e. does not include the
// component params
//
// and returns a map of key-value param pairs for each component identified.
func GetAllEnvironmentParams(snippet string) (map[string]Params, error) {
return getAllEnvironmentParams(snippet)
}
// SetEnvironmentParams takes
//
// component: the name of the new component to be modified.
......@@ -57,6 +79,6 @@ func SetComponentParams(component, snippet string, params map[string]string) (st
// params: the parameters to be set for 'component'.
//
// and returns the jsonnet snippet with the modified set of environment parameters.
func SetEnvironmentParams(component, snippet string, params map[string]string) (string, error) {
func SetEnvironmentParams(component, snippet string, params Params) (string, error) {
return setEnvironmentParams(component, snippet, params)
}
......@@ -13,7 +13,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package snippet
package params
import (
"bytes"
......@@ -39,30 +39,44 @@ func astRoot(component, snippet string) (ast.Node, error) {
return parser.Parse(tokens)
}
func visitParams(component ast.Node) (map[string]string, *ast.LocationRange, error) {
params := make(map[string]string)
func visitParams(component ast.Node) (Params, *ast.LocationRange, error) {
params := make(Params)
var loc *ast.LocationRange
switch n := component.(type) {
case *ast.Object:
loc = n.Loc()
for _, field := range n.Fields {
if field.Id != nil {
key := string(*field.Id)
val, err := visitParamValue(field.Expr2)
if err != nil {
return nil, nil, err
}
params[key] = val
n, isObj := component.(*ast.Object)
if !isObj {
return nil, nil, fmt.Errorf("Expected component node type to be object")
}
loc = n.Loc()
for _, field := range n.Fields {
if field.Id != nil {
key := string(*field.Id)
val, err := visitParamValue(field.Expr2)
if err != nil {
return nil, nil, err
}
params[key] = val
}
default:
return nil, nil, fmt.Errorf("Expected component node type to be object")
}
return params, loc, nil
}
func visitAllParams(components ast.Object) (map[string]Params, error) {
params := make(map[string]Params)
for _, f := range components.Fields {
p, _, err := visitParams(f.Expr2)
if err != nil {
return nil, err
}
params[string(*f.Id)] = p
}
return params, nil
}
// visitParamValue returns a string representation of the param value, quoted
// where necessary. Currently only handles trivial types, ex: string, int, bool
func visitParamValue(param ast.Node) (string, error) {
......@@ -78,7 +92,7 @@ func visitParamValue(param ast.Node) (string, error) {
}
}
func writeParams(indent int, params map[string]string) string {
func writeParams(indent int, params Params) string {
// keys maintains an alphabetically sorted list of the param keys
keys := make([]string, 0, len(params))
for key := range params {
......@@ -106,41 +120,41 @@ func writeParams(indent int, params map[string]string) string {
// ---------------------------------------------------------------------------
// Component Parameter-specific functionality
func visitComponentsObj(component, snippet string) (*ast.Node, error) {
func visitComponentsObj(component, snippet string) (*ast.Object, error) {
root, err := astRoot(component, snippet)
if err != nil {
return nil, err
}
switch n := root.(type) {
case *ast.Object:
for _, field := range n.Fields {
if field.Id != nil && *field.Id == componentsID {
return &field.Expr2, nil
n, isObj := root.(*ast.Object)
if !isObj {
return nil, fmt.Errorf("Invalid format; expected to find a top-level object")
}
for _, field := range n.Fields {
if field.Id != nil && *field.Id == componentsID {
c, isObj := field.Expr2.(*ast.Object)
if !isObj {
return nil, fmt.Errorf("Expected components node type to be object")
}
return c, nil
}
}
// If this point has been reached, it means we weren't able to find a top-level components object.
return nil, fmt.Errorf("Invalid format; expected to find a top-level components object")
}
func appendComponent(component, snippet string, params map[string]string) (string, error) {
func appendComponent(component, snippet string, params Params) (string, error) {
componentsNode, err := visitComponentsObj(component, snippet)
if err != nil {
return "", err
}
// Find the location to append the next component
switch n := (*componentsNode).(type) {
case *ast.Object:
// Ensure that the component we are trying to create params for does not already exist.
for _, field := range n.Fields {
if field.Id != nil && string(*field.Id) == component {
return "", fmt.Errorf("Component parameters for '%s' already exists", component)
}
// Ensure that the component we are trying to create params for does not already exist.
for _, field := range componentsNode.Fields {
if field.Id != nil && string(*field.Id) == component {
return "", fmt.Errorf("Component parameters for '%s' already exists", component)
}
default:
return "", fmt.Errorf("Expected components node type to be object")
}
lines := strings.Split(snippet, "\n")
......@@ -160,27 +174,31 @@ func appendComponent(component, snippet string, params map[string]string) (strin
return strings.Join(lines, "\n"), nil
}
func getComponentParams(component, snippet string) (map[string]string, *ast.LocationRange, error) {
func getComponentParams(component, snippet string) (Params, *ast.LocationRange, error) {
componentsNode, err := visitComponentsObj(component, snippet)
if err != nil {
return nil, nil, err
}
switch n := (*componentsNode).(type) {
case *ast.Object:
for _, field := range n.Fields {
if field.Id != nil && string(*field.Id) == component {
return visitParams(field.Expr2)
}
for _, field := range componentsNode.Fields {
if field.Id != nil && string(*field.Id) == component {
return visitParams(field.Expr2)
}
default:
return nil, nil, fmt.Errorf("Expected component node type to be object")
}
return nil, nil, fmt.Errorf("Could not find component identifier '%s' when attempting to set params", component)
}
func setComponentParams(component, snippet string, params map[string]string) (string, error) {
func getAllComponentParams(snippet string) (map[string]Params, error) {
componentsNode, err := visitComponentsObj("", snippet)
if err != nil {
return nil, err
}
return visitAllParams(*componentsNode)
}
func setComponentParams(component, snippet string, params Params) (string, error) {
currentParams, loc, err := getComponentParams(component, snippet)
if err != nil {
return "", err
......@@ -203,7 +221,7 @@ func setComponentParams(component, snippet string, params map[string]string) (st
// ---------------------------------------------------------------------------
// Environment Parameter-specific functionality
func findEnvComponentsObj(node ast.Node) (ast.Node, error) {
func findEnvComponentsObj(node ast.Node) (*ast.Object, error) {
switch n := node.(type) {
case *ast.Local:
return findEnvComponentsObj(n.Body)
......@@ -212,7 +230,11 @@ func findEnvComponentsObj(node ast.Node) (ast.Node, error) {
case *ast.Object:
for _, f := range n.Fields {
if *f.Id == "components" {
return f.Expr2, nil
c, isObj := f.Expr2.(*ast.Object)
if !isObj {
return nil, fmt.Errorf("Expected components node type to be object")
}
return c, nil
}
}
return nil, fmt.Errorf("Invalid params schema -- found %T that is not 'components'", n)
......@@ -220,40 +242,49 @@ func findEnvComponentsObj(node ast.Node) (ast.Node, error) {
return nil, fmt.Errorf("Invalid params schema -- did not expect type: %T", node)
}
func getEnvironmentParams(component, snippet string) (map[string]string, *ast.LocationRange, bool, error) {
func getEnvironmentParams(component, snippet string) (Params, *ast.LocationRange, bool, error) {
root, err := astRoot(component, snippet)
if err != nil {
return nil, nil, false, err
}
componentsNode, err := findEnvComponentsObj(root)
n, err := findEnvComponentsObj(root)
if err != nil {
return nil, nil, false, err
}
switch n := componentsNode.(type) {
case *ast.Object:
for _, f := range n.Fields {
if f.Id != nil && string(*f.Id) == component {
params, loc, err := visitParams(f.Expr2)
return params, loc, true, err
}
}
// If this point has been reached, it's because we don't have the
// component in the list of params, return the location after the
// last field of the components obj
loc := ast.LocationRange{
Begin: ast.Location{Line: n.Loc().End.Line - 1, Column: n.Loc().End.Column},
End: ast.Location{Line: n.Loc().End.Line, Column: n.Loc().End.Column},
for _, f := range n.Fields {
if f.Id != nil && string(*f.Id) == component {
params, loc, err := visitParams(f.Expr2)
return params, loc, true, err
}
}
// If this point has been reached, it's because we don't have the
// component in the list of params, return the location after the
// last field of the components obj
loc := ast.LocationRange{
Begin: ast.Location{Line: n.Loc().End.Line - 1, Column: n.Loc().End.Column},
End: ast.Location{Line: n.Loc().End.Line, Column: n.Loc().End.Column},
}
return make(Params), &loc, false, nil
}
return make(map[string]string), &loc, false, nil
func getAllEnvironmentParams(snippet string) (map[string]Params, error) {
root, err := astRoot("", snippet)
if err != nil {
return nil, err
}
componentsNode, err := findEnvComponentsObj(root)
if err != nil {
return nil, err
}
return nil, nil, false, fmt.Errorf("Could not find component identifier '%s' when attempting to set params", component)
return visitAllParams(*componentsNode)
}
func setEnvironmentParams(component, snippet string, params map[string]string) (string, error) {
func setEnvironmentParams(component, snippet string, params Params) (string, error) {
currentParams, loc, hasComponent, err := getEnvironmentParams(component, snippet)
if err != nil {
return "", err
......
......@@ -13,7 +13,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package snippet
package params
import (
"reflect"
......@@ -24,7 +24,7 @@ func TestAppendComponentParams(t *testing.T) {
tests := []struct {