Merge pull request #338 from bryanl/upgrader

ksonnet app.yaml format changes in next minor release. Handle both versions

Gopkg.lock

@@ -255,6 +255,18 @@
   packages = ["."]
   revision = "fc9e8d8ef48496124e79ae0df75490096eccf6fe"
 
+[[projects]]
+  name = "github.com/mattn/go-runewidth"
+  packages = ["."]
+  revision = "9e777a8366cce605130a531d2cd6363d07ad7317"
+  version = "v0.0.2"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/olekukonko/tablewriter"
+  packages = ["."]
+  revision = "b8a9be070da40449e501c3c4730a889e42d87a9e"
+
 [[projects]]
   name = "github.com/onsi/ginkgo"
   packages = [
@@ -359,10 +371,17 @@
   packages = ["."]
   revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66"
 
+[[projects]]
+  name = "github.com/stretchr/objx"
+  packages = ["."]
+  revision = "facf9a85c22f48d2f52f2380e4efce1768749a89"
+  version = "v0.1"
+
 [[projects]]
   name = "github.com/stretchr/testify"
   packages = [
     "assert",
+    "mock",
     "require"
   ]
   revision = "f6abca593680b2315d2075e0f5e2a9751e3f431a"
@@ -603,6 +622,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "8b786abd404b80a5933fdf9b29cae67837bf6630041f144bffd7092fdb0e331c"
+  inputs-digest = "cb1ea2fcb93a4e383a77524868cca62997ccf5e8b862f2d8250c388870c4da49"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -32,10 +32,6 @@
   name = "github.com/emicklei/go-restful-swagger12"
   revision = "7524189396c68dc4b04d53852f9edc00f816b123"
 
-[[constraint]]
-  name = "github.com/fatih/color"
-  revision = "5df930a27be2502f99b292b7cc09ebad4d0891f4"
-
 [[constraint]]
   name = "github.com/ghodss/yaml"
   revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7"

Makefile

@@ -17,10 +17,11 @@ VERSION?=dev-$(shell date +%FT%T%z)
 KS_BIN?=ks
 
 APIMACHINERY_VER := $(shell dep status | grep k8s.io/apimachinery | awk '{print $$3}')
+REVISION=$(shell git rev-parse HEAD)
 
 GO = go
 EXTRA_GO_FLAGS =
-GO_FLAGS = -ldflags="-X main.version=$(VERSION) -X main.apimachineryVersion=$(APIMACHINERY_VER) $(GO_LDFLAGS)" $(EXTRA_GO_FLAGS)
+GO_FLAGS = -ldflags="-X main.version=$(VERSION) -X main.apimachineryVersion=$(APIMACHINERY_VER) -X generator.revision=$(REVISION) $(GO_LDFLAGS) " $(EXTRA_GO_FLAGS)
 GOFMT = gofmt
 # GINKGO = "go test" also works if you want to avoid ginkgo tool
 GINKGO = ginkgo

actions/upgrade.go

+package actions
+
+import (
+	"os"
+
+	"github.com/ksonnet/ksonnet/metadata"
+)
+
+// Upgrade upgrades a ksonnet application.
+func Upgrade(dryRun bool) error {
+	cwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+
+	m, err := metadata.Find(cwd)
+	if err != nil {
+		return err
+	}
+
+	a, err := m.App()
+	if err != nil {
+		return err
+	}
+
+	return a.Upgrade(dryRun)
+}

client/client.go

@@ -238,12 +238,12 @@ func (c *Config) overrideCluster(envName string) error {
 	//
 
 	log.Debugf("Validating deployment at '%s' with server '%v'", envName, reflect.ValueOf(servers).MapKeys())
-	env, err := metadataManager.GetEnvironment(envName)
+	destination, err := metadataManager.GetDestination(envName)
 	if err != nil {
 		return err
 	}
 
-	server, err := str.NormalizeURL(env.Destination.Server)
+	server, err := str.NormalizeURL(destination.Server())
 	if err != nil {
 		return err
 	}
@@ -255,11 +255,12 @@ func (c *Config) overrideCluster(envName string) error {
 			c.Overrides.Context.Cluster = clusterName
 		}
 		if c.Overrides.Context.Namespace == "" {
-			log.Debugf("Overwriting --namespace flag with '%s'", env.Destination.Namespace)
-			c.Overrides.Context.Namespace = env.Destination.Namespace
+			log.Debugf("Overwriting --namespace flag with '%s'", destination.Namespace())
+			c.Overrides.Context.Namespace = destination.Namespace()
 		}
 		return nil
 	}
 
-	return fmt.Errorf("Attempting to deploy to environment '%s' at '%s', but cannot locate a server at that address", envName, env.Destination.Server)
+	return fmt.Errorf("Attempting to deploy to environment '%s' at '%s', but cannot locate a server at that address",
+		envName, destination.Server())
 }

cmd/param.go

@@ -17,6 +17,7 @@ package cmd
 
 import (
 	"fmt"
+	"os"
 	"strings"
 
 	"github.com/spf13/cobra"
@@ -27,6 +28,7 @@ import (
 const (
 	flagParamEnv       = "env"
 	flagParamComponent = "component"
+	flagParamNamespace = "namespace"
 )
 
 var paramShortDesc = map[string]string{
@@ -44,6 +46,7 @@ func init() {
 
 	paramSetCmd.PersistentFlags().String(flagParamEnv, "", "Specify environment to set parameters for")
 	paramListCmd.PersistentFlags().String(flagParamEnv, "", "Specify environment to list parameters for")
+	paramListCmd.Flags().String(flagParamNamespace, "", "Specify namespace to list parameters for")
 	paramDiffCmd.PersistentFlags().String(flagParamComponent, "", "Specify the component to diff against")
 }
 
@@ -156,7 +159,12 @@ var paramListCmd = &cobra.Command{
 			return err
 		}
 
-		c := kubecfg.NewParamListCmd(component, env)
+		nsName, err := flags.GetString(flagParamNamespace)
+		if err != nil {
+			return err
+		}
+
+		c := kubecfg.NewParamListCmd(component, env, nsName)
 
 		return c.Run(cmd.OutOrStdout())
 	},
@@ -196,6 +204,11 @@ var paramDiffCmd = &cobra.Command{
 			return fmt.Errorf("'param diff' takes exactly two arguments: the respective names of the environments being diffed")
 		}
 
+		cwd, err := os.Getwd()
+		if err != nil {
+			return err
+		}
+
 		env1 := args[0]
 		env2 := args[1]
 
@@ -204,7 +217,7 @@ var paramDiffCmd = &cobra.Command{
 			return err
 		}
 
-		c := kubecfg.NewParamDiffCmd(env1, env2, component)
+		c := kubecfg.NewParamDiffCmd(appFs, cwd, env1, env2, component)
 
 		return c.Run(cmd.OutOrStdout())
 	},

cmd/pkg.go

@@ -242,7 +242,7 @@ var pkgListCmd = &cobra.Command{
 			return err
 		}
 
-		app, err := manager.AppSpec()
+		app, err := manager.App()
 		if err != nil {
 			return err
 		}
@@ -254,14 +254,14 @@ var pkgListCmd = &cobra.Command{
 				strings.Repeat("=", len(nameHeader)),
 				strings.Repeat("=", len(installedHeader))},
 		}
-		for name := range app.Registries {
+		for name := range app.Registries() {
 			reg, _, err := manager.GetRegistry(name)
 			if err != nil {
 				return err
 			}
 
 			for libName := range reg.Libraries {
-				_, isInstalled := app.Libraries[libName]
+				_, isInstalled := app.Libraries()[libName]
 				if isInstalled {
 					rows = append(rows, []string{name, libName, installed})
 				} else {

cmd/registry.go

@@ -82,7 +82,7 @@ var registryListCmd = &cobra.Command{
 			return err
 		}
 
-		app, err := manager.AppSpec()
+		app, err := manager.App()
 		if err != nil {
 			return err
 		}
@@ -95,7 +95,7 @@ var registryListCmd = &cobra.Command{
 				strings.Repeat("=", len(uriHeader)),
 			},
 		}
-		for name, regRef := range app.Registries {
+		for name, regRef := range app.Registries() {
 			rows = append(rows, []string{name, regRef.Protocol, regRef.URI})
 		}
 
@@ -141,12 +141,12 @@ var registryDescribeCmd = &cobra.Command{
 			return err
 		}
 
-		app, err := manager.AppSpec()
+		app, err := manager.App()
 		if err != nil {
 			return err
 		}
 
-		regRef, exists := app.GetRegistryRef(name)
+		regRef, exists := app.Registries()[name]
 		if !exists {
 			return fmt.Errorf("Registry '%s' doesn't exist", name)
 		}

cmd/root.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 
 	"github.com/ksonnet/ksonnet/component"
+	"github.com/ksonnet/ksonnet/env"
 	"github.com/ksonnet/ksonnet/metadata"
 	"github.com/ksonnet/ksonnet/plugin"
 	str "github.com/ksonnet/ksonnet/strings"
@@ -326,9 +327,14 @@ func (te *cmdObjExpander) Expand() ([]*unstructured.Unstructured, error) {
 		return nil, err
 	}
 
+	app, err := manager.App()
+	if err != nil {
+		return nil, err
+	}
+
 	expander.FlagJpath = append([]string{string(vendorPath), string(libPath), string(envPath)}, expander.FlagJpath...)
 
-	namespacedComponentPaths, err := component.MakePathsByNamespace(te.config.fs, manager, te.config.cwd, te.config.env)
+	namespacedComponentPaths, err := component.MakePathsByNamespace(te.config.fs, app, te.config.cwd, te.config.env)
 	if err != nil {
 		return nil, errors.Wrap(err, "component paths")
 	}
@@ -453,28 +459,20 @@ func importParams(path string) string {
 	return fmt.Sprintf(`%s=import "%s"`, metadata.ParamsExtCodeKey, path)
 }
 
-func importEnv(manager metadata.Manager, env string) (string, error) {
-	app, err := manager.AppSpec()
+func importEnv(manager metadata.Manager, envName string) (string, error) {
+	app, err := manager.App()
 	if err != nil {
 		return "", err
 	}
 
-	spec, exists := app.GetEnvironmentSpec(env)
-	if !exists {
-		return "", fmt.Errorf("Environment '%s' does not exist in app.yaml", env)
-	}
-
-	type EnvironmentSpec struct {
-		Server    string `json:"server"`
-		Namespace string `json:"namespace"`
+	spec, err := app.Environment(envName)
+	if err != nil {
+		return "", fmt.Errorf("Environment '%s' does not exist in app.yaml", envName)
 	}
 
-	toMarshal := &EnvironmentSpec{
-		Server:    spec.Destination.Server,
-		Namespace: spec.Destination.Namespace,
-	}
+	destination := env.NewDestination(spec.Destination.Server, spec.Destination.Namespace)
 
-	marshalled, err := json.Marshal(toMarshal)
+	marshalled, err := json.Marshal(&destination)
 	if err != nil {
 		return "", err
 	}

cmd/upgrade.go

+// Copyright 2018 The ksonnet authors
+//
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+
+package cmd
+
+import (
+	"github.com/ksonnet/ksonnet/actions"
+	"github.com/spf13/cobra"
+)
+
+const (
+	upgradeShortDesc  = "Upgrade ks configuration"
+	flagUpgradeDryRun = "dry-run"
+)
+
+var upgradeCmd = &cobra.Command{
+	Use:   "upgrade [--dry-run]",
+	Short: upgradeShortDesc,
+	Long:  upgradeLong,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		dryRun, err := cmd.Flags().GetBool(flagUpgradeDryRun)
+		if err != nil {
+			return err
+		}
+
+		return actions.Upgrade(dryRun)
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(upgradeCmd)
+
+	upgradeCmd.Flags().Bool(flagUpgradeDryRun, false, "Dry-run upgrade process. Prints out changes.")
+}
+
+const upgradeLong = `
+The upgrade command upgrades a ksonnet application to the latest version.
+
+### Syntax
+
+	Example:
+
+# Upgrade ksonnet application in dry-run mode to see the changes to be performed by the
+# upgrade process.
+ks upgrade --dry-run
+
+# Upgrade ksonnet application. This will update app.yaml to apiVersion 0.1.0
+# and migrate environment spec.json files to ` + "`" + `app.yaml` + "`" + `.
+ks upgrade
+`

component/component.go

@@ -77,6 +77,15 @@ type Namespace struct {
 	fs   afero.Fs
 }
 
+// NewNamespace creates an an instance of Namespace.
+func NewNamespace(fs afero.Fs, root, name string) Namespace {
+	return Namespace{
+		Path: name,
+		root: root,
+		fs:   fs,
+	}
+}
+
 // ExtractNamespacedComponent extracts a namespace and a component from a path.
 func ExtractNamespacedComponent(fs afero.Fs, root, path string) (Namespace, string) {
 	path, component := filepath.Split(path)
@@ -197,15 +206,9 @@ func isComponentDir(fs afero.Fs, path string) (bool, error) {
 	return false, nil
 }
 
-// AppSpecer is implemented by any value that has a AppSpec method. The AppSpec method is
-// used to retrieve a ksonnet AppSpec.
-type AppSpecer interface {
-	AppSpec() (*app.Spec, error)
-}
-
 // MakePathsByNamespace creates a map of component paths categorized by namespace.
-func MakePathsByNamespace(fs afero.Fs, appSpecer AppSpecer, root, env string) (map[Namespace][]string, error) {
-	paths, err := MakePaths(fs, appSpecer, root, env)
+func MakePathsByNamespace(fs afero.Fs, ksApp app.App, root, env string) (map[Namespace][]string, error) {
+	paths, err := MakePaths(fs, ksApp, root, env)
 	if err != nil {
 		return nil, err
 	}
@@ -230,8 +233,8 @@ func MakePathsByNamespace(fs afero.Fs, appSpecer AppSpecer, root, env string) (m
 }
 
 // MakePaths creates a slice of component paths
-func MakePaths(fs afero.Fs, appSpecer AppSpecer, root, env string) ([]string, error) {
-	cpl, err := newComponentPathLocator(fs, appSpecer, env)
+func MakePaths(fs afero.Fs, ksApp app.App, root, env string) ([]string, error) {
+	cpl, err := newComponentPathLocator(fs, ksApp, env)
 	if err != nil {
 		return nil, errors.Wrap(err, "create component path locator")
 	}
@@ -244,22 +247,17 @@ type componentPathLocator struct {
 	envSpec *app.EnvironmentSpec
 }
 
-func newComponentPathLocator(fs afero.Fs, appSpecer AppSpecer, env string) (*componentPathLocator, error) {
-	if appSpecer == nil {
-		return nil, errors.New("appSpecer is nil")
+func newComponentPathLocator(fs afero.Fs, ksApp app.App, env string) (*componentPathLocator, error) {
+	if ksApp == nil {
+		return nil, errors.New("app is nil")
 	}
 
 	if fs == nil {
 		return nil, errors.New("fs is nil")
 	}
 
-	appSpec, err := appSpecer.AppSpec()
+	envSpec, err := ksApp.Environment(env)
 	if err != nil {
-		return nil, errors.Wrap(err, "lookup application spec")
-	}
-
-	envSpec, ok := appSpec.GetEnvironmentSpec(env)
-	if !ok {
 		return nil, errors.Errorf("can't find %s environment", env)
 	}
 

component/component_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/ksonnet/ksonnet/metadata/app"
+	"github.com/ksonnet/ksonnet/metadata/app/mocks"
 )
 
 var (
@@ -48,21 +49,6 @@ var (
 	}
 )
 
-type stubAppSpecer struct {
-	appSpec *app.Spec
-	err     error
-}
-
-var _ AppSpecer = (*stubAppSpecer)(nil)
-
-func newStubAppSpecer(appSpec *app.Spec) *stubAppSpecer {
-	return &stubAppSpecer{appSpec: appSpec}
-}
-
-func (s *stubAppSpecer) AppSpec() (*app.Spec, error) {
-	return s.appSpec, s.err
-}
-
 func makePaths(t *testing.T, fs afero.Fs, paths []string) {
 	for _, path := range paths {
 		dir := filepath.Dir(path)
@@ -311,15 +297,13 @@ func TestMakePathsByNameSpace(t *testing.T) {
 				Targets: tc.targets,
 			}
 
-			appSpec := &app.Spec{
-				Environments: app.EnvironmentSpecs{"default": envSpec},
-			}
-			appSpecer := newStubAppSpecer(appSpec)
+			appMock := &mocks.App{}
+			appMock.On("Environment", "default").Return(envSpec, nil)
 
 			root := "/"
 			env := "default"
 
-			paths, err := MakePathsByNamespace(fs, appSpecer, root, env)
+			paths, err := MakePathsByNamespace(fs, appMock, root, env)
 			if tc.isErr {
 				require.Error(t, err)
 			} else {
@@ -394,15 +378,13 @@ func TestMakePaths(t *testing.T) {
 				Targets: tc.targets,
 			}
 
-			appSpec := &app.Spec{
-				Environments: app.EnvironmentSpecs{"default": envSpec},
-			}
-			appSpecer := newStubAppSpecer(appSpec)
+			appMock := &mocks.App{}
+			appMock.On("Environment", "default").Return(envSpec, nil)
 
 			root := "/"
 			env := "default"
 
-			paths, err := MakePaths(fs, appSpecer, root, env)
+			paths, err := MakePaths(fs, appMock, root, env)
 			if tc.isErr {
 				require.Error(t, err)
 			} else {
@@ -413,14 +395,14 @@ func TestMakePaths(t *testing.T) {
 	}
 }
 
-func TestMakePaths_invalid_appSpecer(t *testing.T) {
+func TestMakePaths_invalid_app(t *testing.T) {
 	fs := afero.NewMemMapFs()
 	_, err := MakePaths(fs, nil, "/", "default")
 	require.Error(t, err)
 }
 
 func TestMakePaths_invalid_fs(t *testing.T) {
-	appSpecer := newStubAppSpecer(nil)
-	_, err := MakePaths(nil, appSpecer, "/", "default")
+	appMock := &mocks.App{}
+	_, err := MakePaths(nil, appMock, "/", "default")
 	require.Error(t, err)
 }

docs/cli-reference/ks.md

@@ -36,6 +36,7 @@ ks [flags]
 * [ks prototype](ks_prototype.md)	 - Instantiate, inspect, and get examples for ksonnet prototypes
 * [ks registry](ks_registry.md)	 - Manage registries for current project
 * [ks show](ks_show.md)	 - Show expanded manifests for a specific environment.
+* [ks upgrade](ks_upgrade.md)	 - Upgrade ks configuration
 * [ks validate](ks_validate.md)	 - Check generated component manifests against the server's API
 * [ks version](ks_version.md)	 - Print version information for this ksonnet binary
 

docs/cli-reference/ks_param_list.md

@@ -42,8 +42,9 @@ ks param list guestbook --env=dev
 ### Options
 
 ```
-      --env string   Specify environment to list parameters for
-  -h, --help         help for list
+      --env string         Specify environment to list parameters for
+  -h, --help               help for list
+      --namespace string   Specify namespace to list parameters for
 ```
 
 ### Options inherited from parent commands

docs/cli-reference/ks_upgrade.md

+## ks upgrade
+
+Upgrade ks configuration
+
+### Synopsis
+
+
+The upgrade command upgrades a ksonnet application to the latest version.
+
+### Syntax
+
+	Example:
+
+# Upgrade ksonnet application in dry-run mode to see the changes to be performed by the
+# upgrade process.
+ks upgrade --dry-run
+
+# Upgrade ksonnet application. This will update app.yaml to apiVersion 0.1.0
+# and migrate environment spec.json files to `app.yaml`.
+ks upgrade
+
+
+```
+ks upgrade [--dry-run] [flags]
+```
+
+### Options
+
+```
+      --dry-run   Dry-run upgrade process. Prints out changes.
+  -h, --help      help for upgrade
+```
+
+### Options inherited from parent commands
+
+```
+  -v, --verbose count[=-1]   Increase verbosity. May be given multiple times.
+```
+
+### SEE ALSO
+
+* [ks](ks.md)	 - Configure your application to deploy to a Kubernetes cluster
+

env/create.go

+package env
+
+import (
+	"fmt"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/ksonnet/ksonnet/metadata/app"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/afero"
+)
+
+// CreateConfig is configuration for creating an environment.
+type CreateConfig struct {
+	App         app.App
+	Destination Destination
+	Fs          afero.Fs
+	K8sSpecFlag string
+	Name        string
+	RootPath    string
+
+	OverrideData []byte
+	ParamsData   []byte
+}
+
+// Create creates a new environment for the project.
+func Create(config CreateConfig) error {
+	c, err := newCreator(config)
+	if err != nil {
+		return err