Add versioned, environment-scoped package support

This change adds version-mapped package support to environments. An environment can specify the specific version of a package to consume - and multiple, side-by-side versions of a package can be installed in the package cache.

Closes #631
Closes #651

* Deprecate GitVersion from LibraryConfig
* pkg.Descriptor.Part -> pkg.Descriptor.Name
* Add Version, Path to PackageManager.
* Composed package structs
* Revendoring - Tailor import path to environment's packages, allow version-free import strings
* Skip unversioned packages when revendoring
* Allow injection of custom importers into jsonnet.VM wrapper
* Allow passing VMOpts to VM via Evaluate*
* Add versioned package evaluation test
* Allow versioned packages to fall back to unversioned paths
* Fix DefaultInstallChecker shadowed variable when looking up environment packages
* Test skipping of missing paths in revendorPackages
* Tweak CacheDependency->ResolveLibrary interface - onFile paths should always be relative to the registry root
* Ensure 0.2.0 version is output when re-writing app.yaml
Signed-off-by: Oren Shomron <shomron@gmail.com>

pkg/actions/pkg_install.go

@@ -72,14 +72,14 @@ func (pi *PkgInstall) Run() error {
 }
 
 func (pi *PkgInstall) parseDepSpec() (pkg.Descriptor, string, error) {
-	d, err := pkg.ParseName(pi.libName)
+	d, err := pkg.Parse(pi.libName)
 	if err != nil {
 		return pkg.Descriptor{}, "", err
 	}
 
 	customName := pi.customName
 	if customName == "" {
-		customName = d.Part
+		customName = d.Name
 	}
 
 	return d, customName, nil

pkg/actions/pkg_install_test.go

@@ -33,7 +33,7 @@ func TestPkgInstall(t *testing.T) {
 		dc := func(a app.App, d pkg.Descriptor, cn string) error {
 			expectedD := pkg.Descriptor{
 				Registry: "incubator",
-				Part:     "apache",
+				Name:     "apache",
 			}
 			require.Equal(t, expectedD, d)
 			require.Equal(t, "customName", cn)

pkg/actions/testdata/env/describe/output.txt

@@ -3,3 +3,4 @@ kubernetesversion: v1.7.0
 path: ""
 destination: null
 targets: []
+libraries: {}

pkg/app/app.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/ksonnet/ksonnet/pkg/lib"
 	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/afero"
 )
 
@@ -100,6 +101,7 @@ type App interface {
 
 // Load loads the application configuration.
 func Load(fs afero.Fs, cwd string, skipFindRoot bool) (App, error) {
+	log := log.WithField("action", "app.Load")
 	appRoot := cwd
 	if !skipFindRoot {
 		var err error
@@ -109,18 +111,32 @@ func Load(fs afero.Fs, cwd string, skipFindRoot bool) (App, error) {
 		}
 	}
 
+	log.Debugf("called")
+
 	spec, err := read(fs, appRoot)
-	if err != nil {
+	if os.IsNotExist(err) {
+		// During `ks init`, app.yaml will not yet exist - generate a new one.
 		return NewApp010(fs, appRoot), nil
 	}
+	if err != nil {
+		return nil, errors.Wrap(err, "reading app configuration")
+	}
 
 	switch spec.APIVersion {
 	default:
 		return nil, errors.Errorf("unknown apiVersion %q in %s", spec.APIVersion, appYamlName)
 	case "0.0.1":
 		return NewApp001(fs, appRoot), nil
-	case "0.1.0":
-		return NewApp010(fs, appRoot), nil
+	case "0.1.0", "0.2.0":
+		// TODO TODO
+		// 0.1.0 will auto-upgraded to 0.2.0. 0.1.0 is read-compatible with
+		// 0.2.0, but will be persisted back as 0.2.0. This behavior will be
+		// subsequently changed with new upgrade framework.
+		a := NewApp010(fs, appRoot)
+		log.Debugf("Upgrading app [%p] version to latest (0.2.0)", a.baseApp)
+		a.config.APIVersion = "0.2.0"
+		a.baseApp.config.APIVersion = "0.2.0"
+		return a, nil
 	}
 }
 

pkg/app/app001.go

@@ -209,7 +209,7 @@ func (a *App001) Upgrade(dryRun bool) error {
 		}
 
 		fmt.Fprintf(a.out, "\n[dry run] Upgraded app.yaml:\n%s\n", string(data))
-		fmt.Fprintf(a.out, "[dry run] You can preform the migration by running `ks upgrade`.\n")
+		fmt.Fprintf(a.out, "[dry run] You can perform the migration by running `ks upgrade`.\n")
 		return nil
 	}
 

pkg/app/app010.go

@@ -139,7 +139,7 @@ func (a *App010) Init() error {
 	}
 
 	msg := "Your application's apiVersion is 0.1.0, but legacy environment declarations " +
-		"where found in environments: %s. In order to proceed, you will have to run `ks upgrade` to " +
+		"were found in environments: %s. In order to proceed, you will have to run `ks upgrade` to " +
 		"upgrade your application. <see url>"
 
 	return errors.Errorf(msg, strings.Join(legacyEnvs, ", "))

pkg/app/base_app.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/ghodss/yaml"
 	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/afero"
 )
 
@@ -70,9 +71,20 @@ func (ba *baseApp) overridePath() string {
 }
 
 func (ba *baseApp) save() error {
+	log := log.WithField("action", "baseApp.save")
+	log.Debugf("called [%p]", ba)
+
 	ba.mu.Lock()
 	defer ba.mu.Unlock()
 
+	if ba.config == nil {
+		return errors.Errorf("cannot save nil app configuration")
+	}
+
+	// Signal we have converted to new app version
+	ba.config.APIVersion = DefaultAPIVersion
+	log.Debugf("saving app version %v", ba.config.APIVersion)
+
 	configData, err := yaml.Marshal(ba.config)
 	if err != nil {
 		return errors.Wrap(err, "convert application configuration to YAML")
@@ -94,6 +106,9 @@ func (ba *baseApp) save() error {
 }
 
 func (ba *baseApp) load() error {
+	log := log.WithField("action", "baseApp.load")
+	log.Debugf("called [%p]", ba)
+
 	ba.mu.Lock()
 	defer ba.mu.Unlock()
 

pkg/app/base_app_test.go

@@ -103,7 +103,7 @@ func Test_baseApp_AddRegistry_override(t *testing.T) {
 	err := ba.AddRegistry(reg, true)
 	require.NoError(t, err)
 
-	assertContents(t, fs, "app010_app.yaml", ba.configPath())
+	assertContents(t, fs, "app020_app.yaml", ba.configPath())
 	assertContents(t, fs, "add-registry-override.yaml", ba.overridePath())
 }
 

pkg/app/override.go

@@ -26,7 +26,7 @@ const (
 	// overrideKind is the override resource type.
 	overrideKind = "ksonnet.io/app-override"
 	// overrideVersion is the version of the override resource.
-	overrideVersion = "0.1.0"
+	overrideVersion = "0.2.0"
 )
 
 // Override defines overrides to ksonnet project configurations.
@@ -52,7 +52,7 @@ func (o *Override) Validate() error {
 
 // IsDefined returns true if the override has environments or registries defined.
 func (o *Override) IsDefined() bool {
-	return len(o.Environments) > 0 || len(o.Registries) > 0
+	return o != nil && (len(o.Environments) > 0 || len(o.Registries) > 0)
 }
 
 // SaveOverride saves the override to the filesystem.

pkg/app/schema.go

@@ -29,7 +29,7 @@ import (
 
 const (
 	// DefaultAPIVersion is the default ks API version to use if not specified.
-	DefaultAPIVersion = "0.1.0"
+	DefaultAPIVersion = "0.2.0"
 	// Kind is the schema resource type.
 	Kind = "ksonnet.io/app"
 	// DefaultVersion is the default version of the app schema.
@@ -49,6 +49,21 @@ var (
 	ErrEnvironmentNotExists = fmt.Errorf("Environment with name doesn't exist")
 )
 
+var (
+	compatibleAPIRangeStrings = []string{
+		">= 0.0.1 <= 0.2.0",
+	}
+	compatibleAPIRanges = mustCompileRanges()
+)
+
+func mustCompileRanges() []semver.Range {
+	result := make([]semver.Range, 0, len(compatibleAPIRangeStrings))
+	for _, s := range compatibleAPIRangeStrings {
+		result = append(result, semver.MustParseRange(s))
+	}
+	return result
+}
+
 // Spec defines all the ksonnet project metadata. This includes details such as
 // the project name, authors, environments, and registries.
 type Spec struct {
@@ -240,11 +255,16 @@ func (r *RegistryConfigs) UnmarshalJSON(b []byte) error {
 	}
 
 	// Set Name fields according to map keys
+	result := RegistryConfigs{}
 	for k, v := range registries {
+		if v == nil {
+			continue
+		}
 		v.Name = k
+		result[k] = v
 	}
 
-	*r = RegistryConfigs(registries)
+	*r = result
 	return nil
 }
 
@@ -261,11 +281,16 @@ func (e *EnvironmentConfigs) UnmarshalJSON(b []byte) error {
 	}
 
 	// Set Name fields according to map keys
+	result := EnvironmentConfigs{}
 	for k, v := range envs {
+		if v == nil {
+			continue
+		}
 		v.Name = k
+		result[k] = v
 	}
 
-	*e = EnvironmentConfigs(envs)
+	*e = result
 	return nil
 }
 
@@ -284,6 +309,8 @@ type EnvironmentConfig struct {
 	// Targets contain the relative component paths that this environment
 	// wishes to deploy on it's destination.
 	Targets []string `json:"targets,omitempty"`
+	// Libraries specifies versioned libraries specifically used by this environment.
+	Libraries LibraryConfigs `json:"libraries,omitempty"`
 
 	isOverride bool
 }
@@ -313,6 +340,13 @@ type EnvironmentDestinationSpec struct {
 
 // LibraryConfig is the specification for a library part.
 type LibraryConfig struct {
+	Name     string `json:"name"`
+	Registry string `json:"registry"`
+	Version  string `json:"version"`
+}
+
+// 0.1.0 version of LibraryConfig
+type libraryConfigDeprecated struct {
 	Name       string          `json:"name"`
 	Registry   string          `json:"registry"`
 	Version    string          `json:"version"`
@@ -328,6 +362,61 @@ type GitVersionSpec struct {
 // LibraryConfigs is a mapping of a library configurations by name.
 type LibraryConfigs map[string]*LibraryConfig
 
+// UnmarshalJSON implements the json.Unmarshaler interface.
+// We implement some compatibility conversions.
+func (l *LibraryConfigs) UnmarshalJSON(b []byte) error {
+	var cfgs map[string]*LibraryConfig
+
+	if err := json.Unmarshal(b, &cfgs); err != nil {
+		return err
+	}
+
+	result := LibraryConfigs{}
+	for k, v := range cfgs {
+		if v == nil {
+			continue
+		}
+
+		v.Name = k
+		result[k] = v
+	}
+
+	*l = result
+	return nil
+}
+
+// libraryConfig is an alias that allows us to leverage default JSON decoding
+// in our custom UnmarshalJSON handler without triggering infinite recursion.
+type libraryConfig LibraryConfig
+
+// UnmarshalJSON implements the json.Unmarshaler interface.
+// We implement some compatibility conversions.
+func (l *LibraryConfig) UnmarshalJSON(b []byte) error {
+	var cfg libraryConfig
+
+	if err := json.Unmarshal(b, &cfg); err != nil {
+		return err
+	}
+	*l = LibraryConfig(cfg)
+
+	// Check if there's any need for conversions
+	if cfg.Version != "" {
+		return nil
+	}
+
+	// Try to convert deprecated fields
+	var oldStyle libraryConfigDeprecated
+	if err := json.Unmarshal(b, &oldStyle); err != nil {
+		// This is best-effort, not an error
+		return nil
+	}
+	if oldStyle.GitVersion != nil {
+		l.Version = oldStyle.GitVersion.CommitSHA
+	}
+
+	return nil
+}
+
 // ContributorSpec is a specification for the project contributors.
 type ContributorSpec struct {
 	Name  string `json:"name"`
@@ -390,17 +479,23 @@ func (s *Spec) validate() error {
 		return errors.New("invalid version")
 	}
 
-	compatVer, _ := semver.Make(DefaultAPIVersion)
 	ver, err := semver.Make(s.APIVersion)
 	if err != nil {
 		return errors.Wrap(err, "Failed to parse version in app spec")
 	}
 
-	if compatVer.Compare(ver) < 0 {
+	var compatible bool
+	for _, compatRange := range compatibleAPIRanges {
+		if compatRange(ver) {
+			compatible = true
+		}
+	}
+
+	if !compatible {
 		return fmt.Errorf(
 			"Current app uses unsupported spec version '%s' (this client only supports %s)",
 			s.APIVersion,
-			DefaultAPIVersion)
+			compatibleAPIRangeStrings)
 	}
 
 	return nil

pkg/app/schema_test.go

@@ -188,7 +188,7 @@ func TestGetEnvironmentConfigs(t *testing.T) {
 // Verifies that EnvironmentConfig.Name fields are injected at unmarshal time.
 func TestEnvironmentConfigHasName(t *testing.T) {
 	b := []byte(`
-apiVersion: 0.1.0
+apiVersion: 0.2.0
 environments:
   default:
     destination:
@@ -382,7 +382,7 @@ func Test_write(t *testing.T) {
 	fs := afero.NewMemMapFs()
 
 	spec := &Spec{
-		APIVersion: "0.1.0",
+		APIVersion: "0.2.0",
 		Environments: EnvironmentConfigs{
 			"a": &EnvironmentConfig{},
 			"b": &EnvironmentConfig{isOverride: true},
@@ -407,7 +407,7 @@ func Test_write_no_override(t *testing.T) {
 	fs := afero.NewMemMapFs()
 
 	spec := &Spec{
-		APIVersion: "0.1.0",
+		APIVersion: "0.2.0",
 		Environments: EnvironmentConfigs{
 			"a": &EnvironmentConfig{},
 		},
@@ -435,7 +435,7 @@ func Test_read(t *testing.T) {
 	require.NoError(t, err)
 
 	expected := &Spec{
-		APIVersion:   "0.1.0",
+		APIVersion:   "0.2.0",
 		Contributors: ContributorSpecs{},
 		Environments: EnvironmentConfigs{
 			"a": &EnvironmentConfig{Name: "a"},
@@ -463,9 +463,17 @@ func TestEnvironmentSpec_MakePath(t *testing.T) {
 }
 
 // Test that RegistryConfigs are properly deserialized, specifically
-// their Name fields, which are handler by customer UnmarshalJSON code.
+// their Name fields, which are handler by custom UnmarshalJSON code.
 func TestUnmarshalRegistryConfigs(t *testing.T) {
-	input := []byte(`
+	tests := []struct {
+		name               string
+		input              string
+		expectedRegistries int
+	}{
+		{
+			name:               "0.1.0",
+			expectedRegistries: 3,
+			input: `
 apiVersion: 0.1.0
 registries:
   incubator:
@@ -481,15 +489,38 @@ registries:
     protocol: github
     uri: github.com/ksonnet/parts/tree/next/incubator
 version: 0.0.1
-`)
-	var spec Spec
+`,
+		},
+		{
+			name:               "0.2.0",
+			expectedRegistries: 3,
+			input: `
+apiVersion: 0.2.0
+registries:
+  incubator:
+    protocol: github
+    uri: github.com/ksonnet/parts/tree/master/incubator
+  helm-stable:
+    protocol: helm
+    uri: https://kubernetes-charts.storage.googleapis.com
+  otherRegistry:
+    protocol: github
+    uri: github.com/ksonnet/parts/tree/next/incubator
+version: 0.0.1
+`,
+		},
+	}
 
-	err := yaml.Unmarshal(input, &spec)
-	assert.NoError(t, err)
-	assert.Equal(t, 3, len(spec.Registries))
+	for _, tc := range tests {
+		var spec Spec
 
-	for k, v := range spec.Registries {
-		assert.Equal(t, k, v.Name)
+		err := yaml.Unmarshal([]byte(tc.input), &spec)
+		assert.NoError(t, err, tc.name)
+		assert.Equal(t, tc.expectedRegistries, len(spec.Registries), tc.name)
+
+		for k, v := range spec.Registries {
+			assert.Equal(t, k, v.Name, tc.name)
+		}
 	}
 }
 

pkg/app/testdata/add-registry-override.yaml

-apiVersion: 0.1.0
+apiVersion: 0.2.0
 kind: ksonnet.io/app-override
 registries:
   new:

pkg/app/testdata/add-registry.yaml

-apiVersion: 0.1.0
+apiVersion: 0.2.0
 environments:
   default:
     destination:

pkg/app/testdata/app020_app.yaml

+apiVersion: 0.2.0
+environments:
+  default:
+    destination:
+      namespace: some-namespace
+      server: http://example.com
+    k8sVersion: v1.7.0
+    path: default
+  us-east/test:
+    destination:
+      namespace: some-namespace
+      server: http://example.com
+    k8sVersion: v1.7.0
+    path: us-east/test
+  us-west/prod:
+    destination:
+      namespace: some-namespace
+      server: http://example.com
+    k8sVersion: v1.7.0
+    path: us-west/prod
+  us-west/test:
+    destination:
+      namespace: some-namespace
+      server: http://example.com
+    k8sVersion: v1.7.0
+    path: us-west/test
+kind: ksonnet.io/app
+name: test-get-envs
+registries:
+  incubator:
+    protocol: ""
+    uri: ""
+version: 0.0.1

pkg/app/testdata/write-app.yaml

-apiVersion: 0.1.0
+apiVersion: 0.2.0
 environments:
   a:
     destination: null

pkg/app/testdata/write-override.yaml

-apiVersion: 0.1.0
+apiVersion: 0.2.0
 environments:
   b:
     destination: null

pkg/clicmd/apply.go

@@ -73,7 +73,7 @@ ks apply dev --dry-run
 ks apply dev -c guestbook-ui
 
 # Create or update multiple components in a ksonnet application (e.g. 'guestbook-ui'
-# and 'ngin-depl') for the 'dev' environment. Does not create resources that are
+# and 'nginx-depl') for the 'dev' environment. Does not create resources that are
 # not already present on the cluster.
 #
 # This essentially deploys 'components/guestbook-ui.jsonnet' and

pkg/cluster/ksonnet_object.go

+// Copyright 2018 The ksonnet authors
+//
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+
 package cluster
 
 import (

pkg/cluster/ksonnet_object_test.go

+// Copyright 2018 The ksonnet authors
+//
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+
 package cluster
 
 import (

pkg/cluster/managed_annotation.go

+// Copyright 2018 The ksonnet authors
+//
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+
 package cluster
 
 import (