Add LibManager for managing k8s API and ksonnet-lib metadata

This change will introduce a lib package that handles the reading,
writing, and serialization of the ksonnet-lib and the kubernete's open
API files.

Prior to this change, metadata was stored in an environment's .metadata
directory. This lead to redundant files where environments share the
same kubernetes API version.
Signed-off-by: Jessica Yuen <im.jessicayuen@gmail.com>

cmd/diff.go

@@ -321,8 +321,11 @@ func expandEnvObjs(fs afero.Fs, cmd *cobra.Command, env string, manager metadata
 		return nil, err
 	}
 
-	libPath, vendorPath := manager.LibPaths()
-	metadataPath, mainPath, paramsPath := manager.EnvPaths(env)
+	_, vendorPath := manager.LibPaths()
+	libPath, mainPath, paramsPath, err := manager.EnvPaths(env)
+	if err != nil {
+		return nil, err
+	}
 	componentPaths, err := manager.ComponentPaths()
 	if err != nil {
 		return nil, err
@@ -338,7 +341,7 @@ func expandEnvObjs(fs afero.Fs, cmd *cobra.Command, env string, manager metadata
 		return nil, err
 	}
 
-	expander.FlagJpath = append([]string{string(libPath), string(vendorPath), string(metadataPath)}, expander.FlagJpath...)
+	expander.FlagJpath = append([]string{string(vendorPath), string(libPath)}, expander.FlagJpath...)
 	expander.ExtCodes = append([]string{baseObj, params, envSpec}, expander.ExtCodes...)
 
 	envFiles := []string{string(mainPath)}

cmd/init.go

@@ -77,7 +77,7 @@ var initCmd = &cobra.Command{
 		}
 
 		log.Infof("Creating a new app '%s' at path '%s'", appName, appRoot)
-		c, err := kubecfg.NewInitCmd(appName, appRoot, specFlag, &server, &namespace)
+		c, err := kubecfg.NewInitCmd(appName, appRoot, &specFlag, &server, &namespace)
 		if err != nil {
 			return err
 		}

cmd/root.go

@@ -432,10 +432,13 @@ func (te *cmdObjExpander) Expand() ([]*unstructured.Unstructured, error) {
 		return nil, errors.Wrap(err, "find metadata")
 	}
 
-	libPath, vendorPath := manager.LibPaths()
-	metadataPath, mainPath, paramsPath := manager.EnvPaths(te.config.env)
+	_, vendorPath := manager.LibPaths()
+	libPath, mainPath, paramsPath, err := manager.EnvPaths(te.config.env)
+	if err != nil {
+		return nil, err
+	}
 
-	expander.FlagJpath = append([]string{string(libPath), string(vendorPath), string(metadataPath)}, expander.FlagJpath...)
+	expander.FlagJpath = append([]string{string(vendorPath), string(libPath)}, expander.FlagJpath...)
 
 	componentPaths, err := manager.ComponentPaths()
 	if err != nil {

integration/integration_suite_test.go

@@ -100,6 +100,7 @@ func runKsonnetWith(flags []string, host, ns string) error {
 						Server:    host,
 					},
 				},
+				KubernetesVersion: "v1.7.0",
 			},
 		},
 	}

metadata/component_test.go

@@ -33,14 +33,11 @@ const (
 )
 
 func populateComponentPaths(t *testing.T) *manager {
-	spec, err := parseClusterSpec(fmt.Sprintf("file:%s", blankSwagger), testFS)
-	if err != nil {
-		t.Fatalf("Failed to parse cluster spec: %v", err)
-	}
+	specFlag := fmt.Sprintf("file:%s", blankSwagger)
 
 	appPath := componentsPath
 	reg := newMockRegistryManager("incubator")
-	m, err := initManager("componentPaths", appPath, spec, &mockAPIServer, &mockNamespace, reg, testFS)
+	m, err := initManager("componentPaths", appPath, &specFlag, &mockAPIServer, &mockNamespace, reg, testFS)
 	if err != nil {
 		t.Fatalf("Failed to init cluster spec: %v", err)
 	}

metadata/environment.go

@@ -22,57 +22,42 @@ import (
 	"path/filepath"
 
 	"github.com/ksonnet/ksonnet/metadata/app"
+	"github.com/ksonnet/ksonnet/metadata/lib"
 	str "github.com/ksonnet/ksonnet/strings"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/afero"
 
-	"github.com/ksonnet/ksonnet/generator"
 	param "github.com/ksonnet/ksonnet/metadata/params"
 )
 
 const (
-	defaultEnvName  = "default"
-	metadataDirName = ".metadata"
-
-	// hidden metadata files
-	schemaFilename        = "swagger.json"
-	extensionsLibFilename = "k.libsonnet"
-	k8sLibFilename        = "k8s.libsonnet"
+	defaultEnvName = "default"
 
 	// primary environment files
 	envFileName    = "main.jsonnet"
 	paramsFileName = "params.libsonnet"
-	specFilename   = "spec.json"
 )
 
 var envPaths = []string{
-	// metadata Dir.wh
-	metadataDirName,
 	// environment base override file
 	envFileName,
 	// params file
 	paramsFileName,
-	// spec file
-	specFilename,
 }
 
-func (m *manager) CreateEnvironment(name, server, namespace string, spec ClusterSpec) error {
-	b, err := spec.OpenAPI()
+func (m *manager) CreateEnvironment(name, server, namespace, k8sSpecFlag string) error {
+	// generate the lib data for this kubernetes version
+	libManager, err := lib.NewManagerWithSpec(k8sSpecFlag, m.appFS, m.libPath)
 	if err != nil {
 		return err
 	}
 
-	kl, err := generator.Ksonnet(b)
-	if err != nil {
-		log.Debugf("Failed to write '%s'", specFilename)
+	if err := libManager.GenerateLibData(); err != nil {
 		return err
 	}
 
-	return m.createEnvironment(name, server, namespace, kl.K, kl.K8s, kl.Swagger)
-}
-
-func (m *manager) createEnvironment(name, server, namespace string, extensionsLibData, k8sLibData, specData []byte) error {
+	// add the environment to the app spec
 	appSpec, err := m.AppSpec()
 	if err != nil {
 		return err
@@ -99,33 +84,10 @@ func (m *manager) createEnvironment(name, server, namespace string, extensionsLi
 		return err
 	}
 
-	metadataPath := str.AppendToPath(envPath, metadataDirName)
-	err = m.appFS.MkdirAll(metadataPath, defaultFolderPermissions)
-	if err != nil {
-		return err
-	}
-
-	log.Infof("Generating environment metadata at path '%s'", envPath)
-
 	metadata := []struct {
 		path string
 		data []byte
 	}{
-		{
-			// schema file
-			str.AppendToPath(metadataPath, schemaFilename),
-			specData,
-		},
-		{
-			// k8s file
-			str.AppendToPath(metadataPath, k8sLibFilename),
-			k8sLibData,
-		},
-		{
-			// extensions file
-			str.AppendToPath(metadataPath, extensionsLibFilename),
-			extensionsLibData,
-		},
 		{
 			// environment base override file
 			str.AppendToPath(envPath, envFileName),
@@ -157,7 +119,7 @@ func (m *manager) createEnvironment(name, server, namespace string, extensionsLi
 				Namespace: namespace,
 			},
 		},
-		// TODO specify k8s version once metadata is moved.
+		KubernetesVersion: libManager.K8sVersion,
 	})
 
 	if err != nil {
@@ -391,6 +353,31 @@ func (m *manager) SetEnvironmentParams(env, component string, params param.Param
 	return nil
 }
 
+func (m *manager) EnvPaths(env string) (libPath, mainPath, paramsPath string, err error) {
+	app, err := m.AppSpec()
+	if err != nil {
+		return
+	}
+
+	envSpec, ok := app.GetEnvironmentSpec(env)
+	if !ok {
+		err = fmt.Errorf("Environment '%s' does not exist", env)
+		return
+	}
+
+	libManager := lib.NewManager(envSpec.KubernetesVersion, m.appFS, m.libPath)
+
+	envPath := str.AppendToPath(m.environmentsPath, env)
+
+	// main.jsonnet file
+	mainPath = str.AppendToPath(envPath, envFileName)
+	// params.libsonnet file
+	paramsPath = str.AppendToPath(envPath, componentParamsFile)
+	// ksonnet-lib file directory
+	libPath, err = libManager.GetLibPath()
+	return
+}
+
 func (m *manager) tryMvEnvDir(dirPathOld, dirPathNew string) error {
 	// first ensure none of these paths exists in the new directory
 	for _, p := range envPaths {
@@ -450,7 +437,7 @@ func (m *manager) generateOverrideData() []byte {
 
 	var buf bytes.Buffer
 	buf.WriteString(fmt.Sprintf("local base = import \"%s\";\n", relBaseLibsonnetPath))
-	buf.WriteString(fmt.Sprintf("local k = import \"%s\";\n\n", extensionsLibFilename))
+	buf.WriteString(fmt.Sprintf("local k = import \"%s\";\n\n", lib.ExtensionsLibFilename))
 	buf.WriteString("base + {\n")
 	buf.WriteString("  // Insert user-specified overrides here. For example if a component is named \"nginx-deployment\", you might have something like:\n")
 	buf.WriteString("  //   \"nginx-deployment\"+: k.deployment.mixin.metadata.labels({foo: \"bar\"})\n")

metadata/environment_test.go

@@ -45,13 +45,10 @@ func mockEnvironments(t *testing.T, appName string) *manager {
 }
 
 func mockEnvironmentsWith(t *testing.T, appName string, envNames []string) *manager {
-	spec, err := parseClusterSpec(fmt.Sprintf("file:%s", blankSwagger), testFS)
-	if err != nil {
-		t.Fatalf("Failed to parse cluster spec: %v", err)
-	}
+	specFlag := fmt.Sprintf("file:%s", blankSwagger)
 
 	reg := newMockRegistryManager("incubator")
-	m, err := initManager(appName, appName, spec, &mockAPIServer, &mockNamespace, reg, testFS)
+	m, err := initManager(appName, appName, &specFlag, &mockAPIServer, &mockNamespace, reg, testFS)
 	if err != nil {
 		t.Fatalf("Failed to init cluster spec: %v", err)
 	}

metadata/interface.go

@@ -38,7 +38,7 @@ var defaultFilePermissions = os.FileMode(0644)
 type Manager interface {
 	Root() string
 	LibPaths() (libPath, vendorPath string)
-	EnvPaths(env string) (metadataPath, mainPath, paramsPath string)
+	EnvPaths(env string) (libPath, mainPath, paramsPath string, err error)
 
 	// Components API.
 	ComponentPaths() ([]string, error)
@@ -58,7 +58,7 @@ type Manager interface {
 	SetEnvironmentParams(env, component string, params param.Params) error
 
 	// Environment API.
-	CreateEnvironment(name, uri, namespace string, spec ClusterSpec) error
+	CreateEnvironment(name, uri, namespace, spec string) error
 	DeleteEnvironment(name string) error
 	GetEnvironments() (app.EnvironmentSpecs, error)
 	GetEnvironment(name string) (*app.EnvironmentSpec, error)
@@ -84,10 +84,8 @@ func Find(path string) (Manager, error) {
 	return findManager(path, afero.NewOsFs())
 }
 
-// Init will retrieve a cluster API specification, generate a
-// capabilities-compliant version of ksonnet-lib, and then generate the
-// directory tree for an application.
-func Init(name, rootPath string, spec ClusterSpec, serverURI, namespace *string) (Manager, error) {
+// Init will generate the directory tree for a ksonnet project.
+func Init(name, rootPath string, k8sSpecFlag, serverURI, namespace *string) (Manager, error) {
 	// Generate `incubator` registry. We do this before before creating
 	// directory tree, in case the network call fails.
 	const (
@@ -104,24 +102,7 @@ func Init(name, rootPath string, spec ClusterSpec, serverURI, namespace *string)
 		return nil, err
 	}
 
-	return initManager(name, rootPath, spec, serverURI, namespace, gh, appFS)
-}
-
-// ClusterSpec represents the API supported by some cluster. There are several
-// ways to specify a cluster, including: querying the API server, reading an
-// OpenAPI spec in some file, or consulting the OpenAPI spec released in a
-// specific version of Kubernetes.
-type ClusterSpec interface {
-	OpenAPI() ([]byte, error)
-	Resource() string // For testing parsing logic.
-}
-
-// ParseClusterSpec will parse a cluster spec flag and output a well-formed
-// ClusterSpec object. For example, if the flag is `--version:v1.7.1`, then we
-// will output a ClusterSpec representing the cluster specification associated
-// with the `v1.7.1` build of Kubernetes.
-func ParseClusterSpec(specFlag string) (ClusterSpec, error) {
-	return parseClusterSpec(specFlag, appFS)
+	return initManager(name, rootPath, k8sSpecFlag, serverURI, namespace, gh, appFS)
 }
 
 // isValidName returns true if a name (e.g., for an environment) is valid.

metadata/clusterspec.go → metadata/lib/clusterspec.go

-package metadata
+// Copyright 2017 The ksonnet authors
+//
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+
+package lib
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -14,7 +30,21 @@ const (
 	k8sVersionURLTemplate = "https://raw.githubusercontent.com/kubernetes/kubernetes/%s/api/openapi-spec/swagger.json"
 )
 
-func parseClusterSpec(specFlag string, fs afero.Fs) (ClusterSpec, error) {
+// ClusterSpec represents the API supported by some cluster. There are several
+// ways to specify a cluster, including: querying the API server, reading an
+// OpenAPI spec in some file, or consulting the OpenAPI spec released in a
+// specific version of Kubernetes.
+type ClusterSpec interface {
+	OpenAPI() ([]byte, error)
+	Resource() string // For testing parsing logic.
+	Version() (string, error)
+}
+
+// ParseClusterSpec will parse a cluster spec flag and output a well-formed
+// ClusterSpec object. For example, if the flag is `--version:v1.7.1`, then we
+// will output a ClusterSpec representing the cluster specification associated
+// with the `v1.7.1` build of Kubernetes.
+func ParseClusterSpec(specFlag string, fs afero.Fs) (ClusterSpec, error) {
 	split := strings.SplitN(specFlag, ":", 2)
 	if len(split) <= 1 || split[1] == "" {
 		return nil, fmt.Errorf("Invalid API specification '%s'", specFlag)
@@ -49,6 +79,32 @@ func (cs *clusterSpecFile) Resource() string {
 	return string(cs.specPath)
 }
 
+func (cs *clusterSpecFile) Version() (string, error) {
+	//
+	// Condensed representation of the spec file, containing the minimal
+	// information necessary to retrieve the spec version.
+	//
+	type Info struct {
+		Version string `json:"version"`
+	}
+
+	type Spec struct {
+		Info Info `json:"info"`
+	}
+
+	bytes, err := cs.OpenAPI()
+	if err != nil {
+		return "", err
+	}
+
+	var spec *Spec
+	if err := json.Unmarshal(bytes, &spec); err != nil {
+		return "", err
+	}
+
+	return spec.Info.Version, nil
+}
+
 type clusterSpecLive struct {
 	apiServerURL string
 }
@@ -61,6 +117,10 @@ func (cs *clusterSpecLive) Resource() string {
 	return string(cs.apiServerURL)
 }
 
+func (cs *clusterSpecLive) Version() (string, error) {
+	return "", fmt.Errorf("Retrieving version spec in live cluster is not implemented")
+}
+
 type clusterSpecVersion struct {
 	k8sVersion string
 }
@@ -85,3 +145,7 @@ func (cs *clusterSpecVersion) OpenAPI() ([]byte, error) {
 func (cs *clusterSpecVersion) Resource() string {
 	return string(cs.k8sVersion)
 }
+
+func (cs *clusterSpecVersion) Version() (string, error) {
+	return string(cs.k8sVersion), nil
+}

metadata/clusterspec_test.go → metadata/lib/clusterspec_test.go

-package metadata
+// Copyright 2017 The ksonnet authors
+//
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+
+package lib
 
 import (
 	"path/filepath"
@@ -18,7 +33,7 @@ var successTests = []parseSuccess{
 
 func TestClusterSpecParsingSuccess(t *testing.T) {
 	for _, test := range successTests {
-		parsed, err := parseClusterSpec(test.input, testFS)
+		parsed, err := ParseClusterSpec(test.input, testFS)
 		if err != nil {
 			t.Errorf("Failed to parse spec: %v", err)
 		}
@@ -66,7 +81,7 @@ var failureTests = []parseFailure{
 
 func TestClusterSpecParsingFailure(t *testing.T) {
 	for _, test := range failureTests {
-		_, err := parseClusterSpec(test.input, testFS)
+		_, err := ParseClusterSpec(test.input, testFS)
 		if err == nil {
 			t.Errorf("Cluster spec parse for '%s' should have failed, but succeeded", test.input)
 		} else if msg := err.Error(); msg != test.errorMsg {

metadata/lib/lib.go

+// Copyright 2017 The ksonnet authors
+//
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+
+package lib
+
+import (
+	"fmt"
+	"os"
+	"path"
+
+	str "github.com/ksonnet/ksonnet/strings"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/afero"
+
+	"github.com/ksonnet/ksonnet/generator"
+)
+
+const (
+	schemaFilename = "swagger.json"
+	k8sLibFilename = "k8s.libsonnet"
+
+	// ExtensionsLibFilename is the file name with the contents of the
+	// generated ksonnet-lib
+	ExtensionsLibFilename = "k.libsonnet"
+)
+
+// Manager operates on the files in the lib directory of a ksonnet project.
+// This included generating the ksonnet-lib files needed to compile the
+// ksonnet (jsonnet) code.
+type Manager struct {
+	// K8sVersion is the Kubernetes version of the Open API spec.
+	K8sVersion string
+
+	spec    ClusterSpec
+	libPath string
+	fs      afero.Fs
+}
+
+// NewManager creates a crew instance of lib.Manager
+func NewManager(k8sVersion string, fs afero.Fs, libPath string) *Manager {
+	return &Manager{K8sVersion: k8sVersion, fs: fs, libPath: libPath}
+}
+
+// NewManagerWithSpec creates a new instance of lib.Manager with the cluster spec initialized.
+func NewManagerWithSpec(k8sSpecFlag string, fs afero.Fs, libPath string) (*Manager, error) {
+	//
+	// Generate the program text for ksonnet-lib.
+	//
+	spec, err := ParseClusterSpec(k8sSpecFlag, fs)
+	if err != nil {
+		return nil, err
+	}
+
+	version, err := spec.Version()
+	if err != nil {
+		return nil, err
+	}
+
+	return &Manager{K8sVersion: version, fs: fs, libPath: libPath, spec: spec}, nil
+}
+
+// GenerateLibData will generate the swagger and ksonnet-lib files in the lib
+// directory of a ksonnet project. The swagger and ksonnet-lib files are
+// unique to each Kubernetes API version. If the files already exist for a
+// specific Kubernetes API version, they won't be re-generated here.
+func (m *Manager) GenerateLibData() error {
+	if m.spec == nil {
+		return fmt.Errorf("Uninitialized ClusterSpec")
+	}
+
+	b, err := m.spec.OpenAPI()
+	if err != nil {
+		return err
+	}
+
+	kl, err := generator.Ksonnet(b)
+	if err != nil {
+		return err
+	}
+
+	versionPath := str.AppendToPath(m.libPath, m.K8sVersion)
+	ok, err := afero.DirExists(m.fs, string(versionPath))
+	if err != nil {
+		return err
+	}
+	if ok {
+		// Already have lib data for this k8s api version
+		return nil
+	}
+
+	err = m.fs.MkdirAll(string(versionPath), os.FileMode(0755))
+	if err != nil {
+		return err
+	}
+
+	files := []struct {
+		path string
+		data []byte
+	}{
+		{
+			// schema file
+			str.AppendToPath(versionPath, schemaFilename),
+			kl.Swagger,
+		},
+		{
+			// k8s file
+			str.AppendToPath(versionPath, k8sLibFilename),
+			kl.K8s,
+		},
+		{
+			// extensions file
+			str.AppendToPath(versionPath, ExtensionsLibFilename),
+			kl.K,
+		},
+	}
+
+	log.Infof("Generating ksonnet-lib data at path '%s'", versionPath)
+
+	for _, a := range files {
+		fileName := path.Base(string(a.path))
+		if err = afero.WriteFile(m.fs, string(a.path), a.data, os.FileMode(0644)); err != nil {
+			log.Debugf("Failed to write '%s'", fileName)
+			return err
+		}
+	}
+
+	return nil
+}
+
+// GetLibPath returns the absolute path pointing to the directory with the
+// metadata files for the provided k8sVersion.
+func (m *Manager) GetLibPath() (string, error) {
+	path := str.AppendToPath(m.libPath, m.K8sVersion)
+	ok, err := afero.DirExists(m.fs, string(path))
+	if err != nil {
+		return "", err
+	}
+	if !ok {
+		return "", fmt.Errorf("Expected lib directory '%s' but was not found", m.K8sVersion)
+	}
+	return path, err
+}

metadata/lib/lib_test.go

+// Copyright 2017 The ksonnet authors
+//
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+package lib
+
+import (
+	"fmt"
+	"os"
+	"testing"
+