First cut at core app metadata management facilities

Much of the tooling build around ksonnet applications will be powered by
metadata presented in a structured directory format. This is similar in
principle to how Hugo and Rails structure web applications.

This commit will begin the process of introducing a FS-based state
machine that manages this directory structure. Primarily, this involves
introducing:

1. Init routines for the directory structure
2. Routines to search parent directories for a ksonnet application
   (similar to how git does this with repositories)

Initially, the directory structures looks like this:

  app-name/
    .ksonnet/   Metadata for ksonnet
    components/ Top-level Kubernetes objects defining application
    lib/        User-written .libsonnet files
    vendor/     Mixin libraries, prototypes

The `.ksonnet` file marks the application root, making it possible to
search parent directories for the root.

As time continues, more verbs will be introduced to manipulate this
metadata (including, e.g., vendoring dependencies, searching prototypes,
and so on).

Makefile

@@ -22,7 +22,7 @@ GOFMT = gofmt
 
 JSONNET_FILES = lib/kubecfg_test.jsonnet examples/guestbook.jsonnet
 # TODO: Simplify this once ./... ignores ./vendor
-GO_PACKAGES = ./cmd/... ./utils/... ./pkg/...
+GO_PACKAGES = ./cmd/... ./utils/... ./pkg/... ./metadata/...
 
 all: kubecfg
 

metadata/clusterspec.go

+package metadata
+
+import (
+	"github.com/spf13/afero"
+)
+
+type clusterSpecFile struct {
+	specPath AbsPath
+}
+
+func (cs *clusterSpecFile) data() ([]byte, error) {
+	return afero.ReadFile(appFS, string(cs.specPath))
+}
+
+func (cs *clusterSpecFile) resource() string {
+	return string(cs.specPath)
+}
+
+type clusterSpecLive struct {
+	apiServerURL string
+}
+
+func (cs *clusterSpecLive) data() ([]byte, error) {
+	// TODO: Implement getting spec from path, k8sVersion, and URL.
+	panic("Not implemented")
+}
+
+func (cs *clusterSpecLive) resource() string {
+	return string(cs.apiServerURL)
+}
+
+type clusterSpecVersion struct {
+	k8sVersion string
+}
+
+func (cs *clusterSpecVersion) data() ([]byte, error) {
+	// TODO: Implement getting spec from path, k8sVersion, and URL.
+	panic("Not implemented")
+}
+
+func (cs *clusterSpecVersion) resource() string {
+	return string(cs.k8sVersion)
+}

metadata/clusterspec_test.go

+package metadata
+
+import (
+	"path/filepath"
+	"testing"
+)
+
+type parseSuccess struct {
+	input  string
+	target ClusterSpec
+}
+
+var successTests = []parseSuccess{
+	{"version:v1.7.1", &clusterSpecVersion{"v1.7.1"}},
+	{"file:swagger.json", &clusterSpecFile{"swagger.json"}},
+	{"url:file:///some_file", &clusterSpecLive{"file:///some_file"}},
+}
+
+func TestClusterSpecParsingSuccess(t *testing.T) {
+	for _, test := range successTests {
+		parsed, err := ParseClusterSpec(test.input)
+		if err != nil {
+			t.Errorf("Failed to parse spec: %v", err)
+		}
+
+		parsedResource := parsed.resource()
+		targetResource := test.target.resource()
+
+		switch pt := parsed.(type) {
+		case *clusterSpecLive:
+		case *clusterSpecVersion:
+			if parsedResource != targetResource {
+				t.Errorf("Expected version '%v', got '%v'", parsedResource, targetResource)
+			}
+		case *clusterSpecFile:
+			// Techncially we're cheating here by passing a *relative path*
+			// into `newPathSpec` instead of an absolute one. This is to
+			// make it work on multiple machines. We convert it here, after
+			// the fact.
+			absPath, err := filepath.Abs(targetResource)
+			if err != nil {
+				t.Errorf("Failed to convert `file:` spec to an absolute path: %v", err)
+			}
+
+			if parsedResource != absPath {
+				t.Errorf("Expected path '%v', got '%v'", absPath, parsedResource)
+			}
+		default:
+			t.Errorf("Unknown cluster spec type '%v'", pt)
+		}
+	}
+}
+
+type parseFailure struct {
+	input    string
+	errorMsg string
+}
+
+var failureTests = []parseFailure{
+	{"fakeprefix:foo", "Could not parse cluster spec 'fakeprefix:foo'"},
+	{"foo:", "Invalid API specification 'foo:'"},
+	{"version:", "Invalid API specification 'version:'"},
+	{"file:", "Invalid API specification 'file:'"},
+	{"url:", "Invalid API specification 'url:'"},
+}
+
+func TestClusterSpecParsingFailure(t *testing.T) {
+	for _, test := range failureTests {
+		_, err := ParseClusterSpec(test.input)
+		if err == nil {
+			t.Errorf("Cluster spec parse for '%s' should have failed, but succeeded", test.input)
+		} else if msg := err.Error(); msg != test.errorMsg {
+			t.Errorf("Expected cluster spec parse error: '%s', got: '%s'", test.errorMsg, msg)
+		}
+	}
+}

metadata/interface.go

+package metadata
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/afero"
+)
+
+// AbsPath is an advisory type that represents an absolute path. It is advisory
+// in that it is not forced to be absolute, but rather, meant to indicate
+// intent, and make code easier to read.
+type AbsPath string
+
+// Manager abstracts over a ksonnet application's metadata, allowing users to do
+// things like: create and delete environments; search for prototypes; vendor
+// libraries; and other non-core-application tasks.
+type Manager interface {
+	Root() AbsPath
+	//
+	// TODO: Fill in methods as we need them.
+	//
+	// GetPrototype(id string) Protoype
+	// SearchPrototypes(query string) []Protoype
+	// VendorLibrary(uri, version string) error
+	// CreateEnv(name string, spec *ClusterSpec) error
+	// DeleteEnv(name string) error
+	//
+}
+
+// Find will recursively search the current directory and its parents for a
+// `.ksonnet` folder, which marks the application root. Returns error if there
+// is no application root.
+func Find(path AbsPath) (Manager, error) {
+	return findManager(path, afero.NewOsFs())
+}
+
+// Init will retrieve a cluster API specification, generate a
+// capabilities-compliant version of ksonnet-lib, and then generate the
+// directory tree for an application.
+func Init(rootPath AbsPath, spec ClusterSpec) (Manager, error) {
+	return initManager(rootPath, spec, afero.NewOsFs())
+}
+
+// ClusterSpec represents the API supported by some cluster. There are several
+// ways to specify a cluster, including: querying the API server, reading an
+// OpenAPI spec in some file, or consulting the OpenAPI spec released in a
+// specific version of Kubernetes.
+type ClusterSpec interface {
+	data() ([]byte, error)
+	resource() string // For testing parsing logic.
+}
+
+// ParseClusterSpec will parse a cluster spec flag and output a well-formed
+// ClusterSpec object. For example, if the flag is `--version:v1.7.1`, then we
+// will output a ClusterSpec representing the cluster specification associated
+// with the `v1.7.1` build of Kubernetes.
+func ParseClusterSpec(specFlag string) (ClusterSpec, error) {
+	split := strings.SplitN(specFlag, ":", 2)
+	if len(split) == 0 || len(split) == 1 || split[1] == "" {
+		return nil, fmt.Errorf("Invalid API specification '%s'", specFlag)
+	}
+
+	switch split[0] {
+	case "version":
+		return &clusterSpecVersion{k8sVersion: split[1]}, nil
+	case "file":
+		abs, err := filepath.Abs(split[1])
+		if err != nil {
+			return nil, err
+		}
+		absPath := AbsPath(abs)
+		return &clusterSpecFile{specPath: absPath}, nil
+	case "url":
+		return &clusterSpecLive{apiServerURL: split[1]}, nil
+	default:
+		return nil, fmt.Errorf("Could not parse cluster spec '%s'", specFlag)
+	}
+}

metadata/manager.go

+package metadata
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+
+	"github.com/spf13/afero"
+)
+
+func appendToAbsPath(originalPath AbsPath, toAppend ...string) AbsPath {
+	paths := append([]string{string(originalPath)}, toAppend...)
+	return AbsPath(path.Join(paths...))
+}
+
+const (
+	defaultEnvName = "dev"
+
+	ksonnetDir    = ".ksonnet"
+	libDir        = "lib"
+	componentsDir = "components"
+	vendorDir     = "vendor"
+	schemaDir     = "vendor/schema"
+	vendorLibDir  = "vendor/lib"
+
+	schemaFilename = "swagger.json"
+)
+
+type manager struct {
+	appFS afero.Fs
+
+	rootPath       AbsPath
+	ksonnetPath    AbsPath
+	libPath        AbsPath
+	componentsPath AbsPath
+	vendorDir      AbsPath
+	schemaDir      AbsPath
+	vendorLibDir   AbsPath
+}
+
+func findManager(abs AbsPath, appFS afero.Fs) (*manager, error) {
+	var lastBase string
+	currBase := string(abs)
+
+	for {
+		currPath := path.Join(currBase, ksonnetDir)
+		exists, err := afero.Exists(appFS, currPath)
+		if err != nil {
+			return nil, err
+		}
+		if exists {
+			return newManager(AbsPath(currBase), appFS), nil
+		}
+
+		lastBase = currBase
+		currBase = filepath.Dir(currBase)
+		if lastBase == currBase {
+			return nil, fmt.Errorf("No ksonnet application found")
+		}
+	}
+}
+
+func initManager(rootPath AbsPath, spec ClusterSpec, appFS afero.Fs) (*manager, error) {
+	data, err := spec.data()
+	if err != nil {
+		return nil, err
+	}
+
+	m := newManager(rootPath, appFS)
+
+	if err = m.createAppDirTree(); err != nil {
+		return nil, err
+	}
+
+	if err = m.cacheClusterSpecData(defaultEnvName, data); err != nil {
+		return nil, err
+	}
+
+	return m, nil
+}
+
+func newManager(rootPath AbsPath, appFS afero.Fs) *manager {
+	return &manager{
+		appFS: appFS,
+
+		rootPath:       rootPath,
+		ksonnetPath:    appendToAbsPath(rootPath, ksonnetDir),
+		libPath:        appendToAbsPath(rootPath, libDir),
+		componentsPath: appendToAbsPath(rootPath, componentsDir),
+		vendorDir:      appendToAbsPath(rootPath, vendorDir),
+		schemaDir:      appendToAbsPath(rootPath, schemaDir),
+		vendorLibDir:   appendToAbsPath(rootPath, vendorLibDir),
+	}
+}
+
+func (m *manager) Root() AbsPath {
+	return m.rootPath
+}
+
+func (m *manager) cacheClusterSpecData(name string, specData []byte) error {
+	envPath := string(appendToAbsPath(m.schemaDir, name))
+	err := m.appFS.MkdirAll(envPath, os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	schemaPath := string(filepath.Join(envPath, schemaFilename))
+	err = afero.WriteFile(m.appFS, schemaPath, specData, os.ModePerm)
+	return err
+}
+
+func (m *manager) createAppDirTree() error {
+	exists, err := afero.DirExists(m.appFS, string(m.rootPath))
+	if err != nil {
+		return fmt.Errorf("Could not check existance of directory '%s':\n%v", m.rootPath, err)
+	} else if exists {
+		return fmt.Errorf("Could not create app; directory '%s' already exists", m.rootPath)
+	}
+
+	paths := []AbsPath{
+		m.rootPath,
+		m.ksonnetPath,
+		m.libPath,
+		m.componentsPath,
+		m.vendorDir,
+		m.schemaDir,
+		m.vendorLibDir,
+	}
+
+	for _, p := range paths {
+		if err := m.appFS.MkdirAll(string(p), os.ModePerm); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

metadata/manager_test.go

+package metadata
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/spf13/afero"
+)
+
+const (
+	blankSwagger     = "/blankSwagger.json"
+	blankSwaggerData = `{}`
+)
+
+var appFS = afero.NewMemMapFs()
+
+func init() {
+	afero.WriteFile(appFS, blankSwagger, []byte(blankSwaggerData), os.ModePerm)
+}
+
+func TestInitSuccess(t *testing.T) {
+	spec, err := ParseClusterSpec(fmt.Sprintf("file:%s", blankSwagger))
+	if err != nil {
+		t.Fatalf("Failed to parse cluster spec: %v", err)
+	}
+
+	appPath := AbsPath("/fromEmptySwagger")
+	_, err = initManager(appPath, spec, appFS)
+	if err != nil {
+		t.Fatalf("Failed to init cluster spec: %v", err)
+	}
+
+	defaultEnvDir := appendToAbsPath(schemaDir, defaultEnvName)
+	paths := []AbsPath{
+		ksonnetDir,
+		libDir,
+		componentsDir,
+		vendorDir,
+		schemaDir,
+		vendorLibDir,
+		defaultEnvDir,
+	}
+
+	for _, p := range paths {
+		path := appendToAbsPath(appPath, string(p))
+		exists, err := afero.DirExists(appFS, string(path))
+		if err != nil {
+			t.Fatalf("Expected to create directory '%s', but failed:\n%v", p, err)
+		} else if !exists {
+			t.Fatalf("Expected to create directory '%s', but failed", path)
+		}
+	}
+
+	envPath := appendToAbsPath(appPath, string(defaultEnvDir))
+	schemaPath := appendToAbsPath(envPath, schemaFilename)
+	bytes, err := afero.ReadFile(appFS, string(schemaPath))
+	if err != nil {
+		t.Fatalf("Failed to read swagger file at '%s':\n%v", schemaPath, err)
+	} else if actualSwagger := string(bytes); actualSwagger != blankSwaggerData {
+		t.Fatalf("Expected swagger file at '%s' to have value: '%s', got: '%s'", schemaPath, blankSwaggerData, actualSwagger)
+	}
+}
+
+func TestFindSuccess(t *testing.T) {
+	findSuccess := func(t *testing.T, appDir, currDir AbsPath) {
+		m, err := findManager(currDir, appFS)
+		if err != nil {
+			t.Fatalf("Failed to find manager at path '%s':\n%v", currDir, err)
+		} else if m.rootPath != appDir {
+			t.Fatalf("Found manager at incorrect path '%s', expected '%s'", m.rootPath, appDir)
+		}
+	}
+
+	spec, err := ParseClusterSpec(fmt.Sprintf("file:%s", blankSwagger))
+	if err != nil {
+		t.Fatalf("Failed to parse cluster spec: %v", err)
+	}
+
+	appPath := AbsPath("/findSuccess")
+	_, err = initManager(appPath, spec, appFS)
+	if err != nil {
+		t.Fatalf("Failed to init cluster spec: %v", err)
+	}
+
+	findSuccess(t, appPath, appPath)
+
+	components := appendToAbsPath(appPath, componentsDir)
+	findSuccess(t, appPath, components)
+
+	// Create empty app file.
+	appFile := appendToAbsPath(components, "app.jsonnet")
+	f, err := appFS.OpenFile(string(appFile), os.O_RDONLY|os.O_CREATE, 0777)
+	if err != nil {
+		t.Fatalf("Failed to touch app file '%s'\n%v", appFile, err)
+	}
+	f.Close()
+
+	findSuccess(t, appPath, appFile)
+}
+
+func TestFindFailure(t *testing.T) {
+	findFailure := func(t *testing.T, currDir AbsPath) {
+		_, err := findManager(currDir, appFS)
+		if err == nil {
+			t.Fatalf("Expected to fail to find ksonnet app in '%s', but succeeded", currDir)
+		}
+	}
+
+	findFailure(t, "/")
+	findFailure(t, "/fakePath")
+	findFailure(t, "")
+}
+
+func TestDoubleNewFailure(t *testing.T) {
+	spec, err := ParseClusterSpec(fmt.Sprintf("file:%s", blankSwagger))
+	if err != nil {
+		t.Fatalf("Failed to parse cluster spec: %v", err)
+	}
+
+	appPath := AbsPath("/doubleNew")
+
+	_, err = initManager(appPath, spec, appFS)
+	if err != nil {
+		t.Fatalf("Failed to init cluster spec: %v", err)
+	}
+
+	targetErr := fmt.Sprintf("Could not create app; directory '%s' already exists", appPath)
+	_, err = initManager(appPath, spec, appFS)
+	if err == nil || err.Error() != targetErr {
+		t.Fatalf("Expected to fail to create app with message '%s', got '%s'", targetErr, err.Error())
+	}
+}