Unverified Commit 56b01610 authored by Oren Shomron's avatar Oren Shomron Committed by GitHub
Browse files

Merge pull request #855 from shomron/qualify-libraries-and-refactor-app



Qualify library names in app.yaml to avoid cross-registry conflicts

Also:

Refactor schema to support explicit migrations

    * Dropped support for 0.0.1 apps
    * Versioning has been pushed up into the Schema types instead of App.
    * Added migrations framework for migrating schema versions, one hop at time

Refactor override handling

    * baseApp.load() / baseApp.save() are override-aware
    * app.read() / app.write() (schema.go) are not - they only serialize/deserialize app.yaml
    * baseApp.load() / baseApp.save() now call app.read() / app.write() instead of duplicating serialization logic
    * Removed isOverride flag from EnvironmentConfig, RegistryConfig
    * Removed override logic from app.Load() - this is handled in baseApp.load() now
    * env set command now respects the --override flag to indicate where to write changes

Closes #830
Closes #849
Closes #617
Signed-off-by: default avatarOren Shomron <shomron@gmail.com>
parents 6f47d32a 2d58dd5a
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
package actions package actions
import ( import (
"bytes"
"crypto/tls" "crypto/tls"
"fmt" "fmt"
"net" "net"
...@@ -24,6 +25,8 @@ import ( ...@@ -24,6 +25,8 @@ import (
"github.com/ksonnet/ksonnet/pkg/app" "github.com/ksonnet/ksonnet/pkg/app"
"github.com/ksonnet/ksonnet/pkg/client" "github.com/ksonnet/ksonnet/pkg/client"
"github.com/ksonnet/ksonnet/pkg/registry"
"github.com/ksonnet/ksonnet/pkg/upgrade"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/spf13/afero" "github.com/spf13/afero"
) )
...@@ -378,7 +381,8 @@ func (o *optionLoader) LoadApp() app.App { ...@@ -378,7 +381,8 @@ func (o *optionLoader) LoadApp() app.App {
} }
if !o.LoadOptionalBool(OptionSkipCheckUpgrade) { if !o.LoadOptionalBool(OptionSkipCheckUpgrade) {
if _, err := a.CheckUpgrade(); err != nil { pm := registry.NewPackageManager(a)
if _, err := upgrade.CheckUpgrade(a, new(bytes.Buffer), pm, false); err != nil {
o.err = errors.Wrap(err, "checking for app upgrades") o.err = errors.Wrap(err, "checking for app upgrades")
return nil return nil
} }
......
...@@ -278,12 +278,11 @@ func mockNsWithName(name string) *cmocks.Module { ...@@ -278,12 +278,11 @@ func mockNsWithName(name string) *cmocks.Module {
return m return m
} }
func mockRegistry(name string, isOverride bool) *rmocks.Registry { func mockRegistry(name string) *rmocks.Registry {
m := &rmocks.Registry{} m := &rmocks.Registry{}
m.On("Name").Return(name) m.On("Name").Return(name)
m.On("Protocol").Return(registry.ProtocolGitHub) m.On("Protocol").Return(registry.ProtocolGitHub)
m.On("URI").Return("github.com/ksonnet/parts/tree/master/incubator") m.On("URI").Return("github.com/ksonnet/parts/tree/master/incubator")
m.On("IsOverride").Return(isOverride)
return m return m
} }
...@@ -38,9 +38,10 @@ func RunEnvList(m map[string]interface{}) error { ...@@ -38,9 +38,10 @@ func RunEnvList(m map[string]interface{}) error {
// EnvList lists available namespaces. To initialize EnvList, // EnvList lists available namespaces. To initialize EnvList,
// use the `NewEnvList` constructor. // use the `NewEnvList` constructor.
type EnvList struct { type EnvList struct {
envListFn func() (app.EnvironmentConfigs, error) envListFn func() (app.EnvironmentConfigs, error)
outputType string envIsOverrideFn func(name string) bool
out io.Writer outputType string
out io.Writer
} }
// NewEnvList creates an instance of EnvList // NewEnvList creates an instance of EnvList
...@@ -55,9 +56,10 @@ func NewEnvList(m map[string]interface{}) (*EnvList, error) { ...@@ -55,9 +56,10 @@ func NewEnvList(m map[string]interface{}) (*EnvList, error) {
} }
el := &EnvList{ el := &EnvList{
outputType: outputType, outputType: outputType,
envListFn: a.Environments, envListFn: a.Environments,
out: os.Stdout, envIsOverrideFn: a.IsEnvOverride,
out: os.Stdout,
} }
return el, nil return el, nil
...@@ -83,7 +85,7 @@ func (el *EnvList) Run() error { ...@@ -83,7 +85,7 @@ func (el *EnvList) Run() error {
for name, env := range environments { for name, env := range environments {
override := "" override := ""
if env.IsOverride() { if el.envIsOverrideFn(name) {
override = "*" override = "*"
} }
......
...@@ -24,6 +24,7 @@ import ( ...@@ -24,6 +24,7 @@ import (
amocks "github.com/ksonnet/ksonnet/pkg/app/mocks" amocks "github.com/ksonnet/ksonnet/pkg/app/mocks"
"github.com/ksonnet/ksonnet/pkg/util/test" "github.com/ksonnet/ksonnet/pkg/util/test"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
...@@ -52,10 +53,12 @@ func TestEnvList(t *testing.T) { ...@@ -52,10 +53,12 @@ func TestEnvList(t *testing.T) {
} }
appMock.On("Environments").Return(envs, nil) appMock.On("Environments").Return(envs, nil)
appMock.On("IsEnvOverride", mock.Anything).Return(false)
} }
envListFail := func(appMock *amocks.App) { envListFail := func(appMock *amocks.App) {
appMock.On("Environments").Return(nil, errors.New("failed")) appMock.On("Environments").Return(nil, errors.New("failed"))
appMock.On("IsEnvOverride", mock.Anything).Return(false)
} }
cases := []struct { cases := []struct {
......
...@@ -61,6 +61,7 @@ type EnvSet struct { ...@@ -61,6 +61,7 @@ type EnvSet struct {
newNsName string newNsName string
newServer string newServer string
newAPISpec string newAPISpec string
isOverride bool
envRenameFn envRenameFn envRenameFn envRenameFn
saveFn saveFn saveFn saveFn
...@@ -77,6 +78,7 @@ func NewEnvSet(m map[string]interface{}) (*EnvSet, error) { ...@@ -77,6 +78,7 @@ func NewEnvSet(m map[string]interface{}) (*EnvSet, error) {
newNsName: ol.LoadOptionalString(OptionNamespace), newNsName: ol.LoadOptionalString(OptionNamespace),
newServer: ol.LoadOptionalString(OptionServer), newServer: ol.LoadOptionalString(OptionServer),
newAPISpec: ol.LoadOptionalString(OptionSpecFlag), newAPISpec: ol.LoadOptionalString(OptionSpecFlag),
isOverride: ol.LoadOptionalBool(OptionOverride),
envRenameFn: env.Rename, envRenameFn: env.Rename,
saveFn: save, saveFn: save,
...@@ -96,11 +98,11 @@ func (es *EnvSet) Run() error { ...@@ -96,11 +98,11 @@ func (es *EnvSet) Run() error {
return err return err
} }
if err := es.updateName(env.IsOverride()); err != nil { if err := es.updateName(es.isOverride); err != nil {
return err return err
} }
if err := es.updateEnvConfig(*env, es.newNsName, es.newServer, es.newAPISpec, env.IsOverride()); err != nil { if err := es.updateEnvConfig(*env, es.newNsName, es.newServer, es.newAPISpec, es.isOverride); err != nil {
return err return err
} }
......
...@@ -155,6 +155,7 @@ func TestEnvSet(t *testing.T) { ...@@ -155,6 +155,7 @@ func TestEnvSet(t *testing.T) {
} }
}, },
}, },
// TODO add tests for overrides here
} }
for _, tc := range cases { for _, tc := range cases {
......
...@@ -71,12 +71,7 @@ func RunPkgRemove(m map[string]interface{}) error { ...@@ -71,12 +71,7 @@ func RunPkgRemove(m map[string]interface{}) error {
// Run removes packages // Run removes packages
func (pr *PkgRemove) Run() error { func (pr *PkgRemove) Run() error {
desc, err := pkg.Parse(pr.pkgName) oldCfg, err := pr.libUpdateFn(pr.pkgName, pr.envName, nil)
if err != nil {
return err
}
oldCfg, err := pr.libUpdateFn(desc.Name, pr.envName, nil)
if err != nil { if err != nil {
return err return err
} }
......
...@@ -41,8 +41,9 @@ type RegistryList struct { ...@@ -41,8 +41,9 @@ type RegistryList struct {
app app.App app app.App
outputType string outputType string
registryListFn func(ksApp app.App) ([]registry.Registry, error) registryListFn func(ksApp app.App) ([]registry.Registry, error)
out io.Writer registryIsOverrideFn func(name string) bool
out io.Writer
} }
// NewRegistryList creates an instance of RegistryList // NewRegistryList creates an instance of RegistryList
...@@ -50,14 +51,20 @@ func NewRegistryList(m map[string]interface{}) (*RegistryList, error) { ...@@ -50,14 +51,20 @@ func NewRegistryList(m map[string]interface{}) (*RegistryList, error) {
ol := newOptionLoader(m) ol := newOptionLoader(m)
httpClient := ol.LoadHTTPClient() httpClient := ol.LoadHTTPClient()
a := ol.LoadApp()
if ol.err != nil {
return nil, ol.err
}
rl := &RegistryList{ rl := &RegistryList{
app: ol.LoadApp(), app: a,
outputType: ol.LoadOptionalString(OptionOutput), outputType: ol.LoadOptionalString(OptionOutput),
registryListFn: func(ksApp app.App) ([]registry.Registry, error) { registryListFn: func(ksApp app.App) ([]registry.Registry, error) {
return registry.List(ksApp, httpClient) return registry.List(ksApp, httpClient)
}, },
out: os.Stdout, registryIsOverrideFn: a.IsRegistryOverride,
out: os.Stdout,
} }
if ol.err != nil { if ol.err != nil {
...@@ -87,7 +94,7 @@ func (rl *RegistryList) Run() error { ...@@ -87,7 +94,7 @@ func (rl *RegistryList) Run() error {
for _, r := range registries { for _, r := range registries {
override := "" override := ""
if r.IsOverride() { if rl.registryIsOverrideFn(r.Name()) {
override = "*" override = "*"
} }
......
...@@ -66,11 +66,17 @@ func TestRegistryList(t *testing.T) { ...@@ -66,11 +66,17 @@ func TestRegistryList(t *testing.T) {
a.registryListFn = func(app.App) ([]registry.Registry, error) { a.registryListFn = func(app.App) ([]registry.Registry, error) {
registries := []registry.Registry{ registries := []registry.Registry{
mockRegistry("override", true), mockRegistry("override"),
mockRegistry("incubator", false), mockRegistry("incubator"),
} }
return registries, nil return registries, nil
} }
a.registryIsOverrideFn = func(name string) bool {
if name == "override" {
return true
}
return false
}
err = a.Run() err = a.Run()
if tc.isErr { if tc.isErr {
......
...@@ -22,7 +22,6 @@ import ( ...@@ -22,7 +22,6 @@ import (
"sort" "sort"
"github.com/pkg/errors" "github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/afero" "github.com/spf13/afero"
) )
...@@ -69,8 +68,10 @@ type App interface { ...@@ -69,8 +68,10 @@ type App interface {
Fs() afero.Fs Fs() afero.Fs
// HTTPClient is the app's http client // HTTPClient is the app's http client
HTTPClient() *http.Client HTTPClient() *http.Client
// CheckUpgrade checks whether an app should be upgraded. // IsEnvOverride returns whether the specified environment has overriding configuration
CheckUpgrade() (bool, error) IsEnvOverride(name string) bool
// IsRegistryOverride returns whether the specified registry has overriding configuration
IsRegistryOverride(name string) bool
// LibPath returns the path of the lib for an environment. // LibPath returns the path of the lib for an environment.
LibPath(envName string) (string, error) LibPath(envName string) (string, error)
// Libraries returns all environments. // Libraries returns all environments.
...@@ -95,42 +96,33 @@ type App interface { ...@@ -95,42 +96,33 @@ type App interface {
UpdateLib(name string, env string, spec *LibraryConfig) (*LibraryConfig, error) UpdateLib(name string, env string, spec *LibraryConfig) (*LibraryConfig, error)
// UpdateRegistry updates a registry. // UpdateRegistry updates a registry.
UpdateRegistry(spec *RegistryConfig) error UpdateRegistry(spec *RegistryConfig) error
// Upgrade upgrades an application to the current version. // Upgrade upgrades an application (app.yaml) to the current version.
Upgrade(dryRun bool) error Upgrade(bool) error
// VendorPath returns the root of the vendor path. // VendorPath returns the root of the vendor path.
VendorPath() string VendorPath() string
} }
// Load loads the application configuration. // Load loads the application configuration.
func Load(fs afero.Fs, httpClient *http.Client, appRoot string) (App, error) { func Load(fs afero.Fs, httpClient *http.Client, appRoot string) (App, error) {
log := log.WithField("action", "app.Load") if fs == nil {
return nil, errors.New("nil fs interface")
}
spec, err := read(fs, appRoot) _, err := fs.Stat(specPath(appRoot))
if os.IsNotExist(err) { if os.IsNotExist(err) {
// During `ks init`, app.yaml will not yet exist - generate a new one. // During `ks init`, app.yaml will not yet exist - generate a new one.
return NewApp010(fs, appRoot, httpClient), nil return NewBaseApp(fs, appRoot, httpClient), nil
} }
if err != nil { if err != nil {
return nil, errors.Wrap(err, "reading app configuration") return nil, errors.Wrap(err, "checking existence of app configuration")
} }
switch spec.APIVersion { a := NewBaseApp(fs, appRoot, httpClient)
default: if err := a.doLoad(); err != nil {
return nil, errors.Errorf("unknown apiVersion %q in %s", spec.APIVersion, appYamlName) return nil, errors.Wrap(err, "reading app configuration")
case "0.0.1":
return NewApp001(fs, appRoot, httpClient), nil
case "0.1.0", "0.2.0":
// TODO TODO
// 0.1.0 will auto-upgraded to 0.2.0. 0.1.0 is read-compatible with
// 0.2.0, but will be persisted back as 0.2.0. This behavior will be
// subsequently changed with new upgrade framework.
a := NewApp010(fs, appRoot, httpClient)
log.Debugf("Interpreting app version as latest (0.2.0)", a.baseApp)
a.config.APIVersion = "0.2.0"
a.baseApp.config.APIVersion = "0.2.0"
return a, nil
} }
return a, nil
} }
func app010LibPath(root string) string { func app010LibPath(root string) string {
......
// Copyright 2018 The kubecfg authors
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package app
import (
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"github.com/ghodss/yaml"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/afero"
)
const (
// app001specJSON is the name for environment schema
app001specJSON = "spec.json"
)
// App001 is a ksonnet 0.0.1 application.
type App001 struct {
out io.Writer
*baseApp
}
var _ App = (*App001)(nil)
// NewApp001 creates an App001 instance.
func NewApp001(fs afero.Fs, root string, httpClient *http.Client) *App001 {
ba := newBaseApp(fs, root, httpClient)
return &App001{
out: os.Stdout,
baseApp: ba,
}
}
// AddEnvironment adds an environment spec to the app spec. If the spec already exists,
// it is overwritten.
func (a *App001) AddEnvironment(e *EnvironmentConfig, k8sSpecFlag string, isOverride bool) error {
if e == nil {
return errors.Errorf("nil environment configuraion")
}
if e.Name == "" {
return errors.Errorf("invalid environment name")
}
// if it is an override, write the destination to override file. If not, do the normal thing.
envPath := filepath.Join(a.root, EnvironmentDirName, e.Name)
if err := a.fs.MkdirAll(envPath, DefaultFolderPermissions); err != nil {
return err
}
specPath := filepath.Join(envPath, app001specJSON)
b, err := json.Marshal(e.Destination)
if err != nil {
return err
}
if err = afero.WriteFile(a.fs, specPath, b, DefaultFilePermissions); err != nil {
return err
}
_, err = a.libUpdater.UpdateKSLib(k8sSpecFlag, a.appLibPath(e.Name))
return err
}
func (a *App001) overrideDestintation(name string, envSpec *EnvironmentConfig) error {
return nil
}
// Environment returns the spec for an environment. In 0.1.0, the file lives in
// /environments/name/spec.json.
func (a *App001) Environment(name string) (*EnvironmentConfig, error) {
path := filepath.Join(a.root, EnvironmentDirName, name, app001specJSON)
return read001EnvSpec(a.fs, name, path)
}
// Environments returns specs for all environments. In 0.1.0, the environment spec
// lives in spec.json files.
func (a *App001) Environments() (EnvironmentConfigs, error) {
specs := EnvironmentConfigs{}
root := filepath.Join(a.root, EnvironmentDirName)
err := afero.Walk(a.fs, root, func(path string, fi os.FileInfo, err error) error {
if fi.IsDir() {
return nil
}
if fi.Name() == app001specJSON {
dir := filepath.Dir(path)
envName := strings.TrimPrefix(dir, root+"/")
spec, err := read001EnvSpec(a.fs, envName, path)
if err != nil {
return err
}
specs[envName] = spec
}
return nil
})
if err != nil {
return nil, err
}
return specs, nil
}
// CheckUpgrade initializes the App.
func (a *App001) CheckUpgrade() (bool, error) {
msg := "Your application's apiVersion is below 0.1.0. In order to use all ks features, you " +
"can upgrade your application using `ks upgrade`."
log.Warn(msg)
return true, nil
}
// LibPath returns the lib path for an env environment.
func (a *App001) LibPath(envName string) (string, error) {
return filepath.Join(a.envDir(envName), ".metadata"), nil
}
// Libraries returns application libraries.
func (a *App001) Libraries() (LibraryConfigs, error) {
if err := a.load(); err != nil {
return nil, errors.Wrap(err, "load configuration")
}
return a.config.Libraries, nil
}
// Registries returns application registries.
func (a *App001) Registries() (RegistryConfigs, error) {
if err := a.load(); err != nil {
return nil, errors.Wrap(err, "load configuration")
}
registries := RegistryConfigs{}
for k, v := range a.config.Registries {
registries[k] = v
}
for k, v := range a.overrides.Registries {
registries[k] = v
}
return registries, nil
}
// RemoveEnvironment removes an environment.
func (a *App001) RemoveEnvironment(envName string, override bool) error {
a.Fs().RemoveAll(a.envDir(envName))
return nil
}
// RenameEnvironment renames environments.
func (a *App001) RenameEnvironment(from, to string, override bool) error {
return moveEnvironment(a.fs, a.root, from, to)
}
// UpdateTargets returns an error since 0.0.1 based applications don't have support
// for targets.
func (a *App001) UpdateTargets(envName string, targets []string) error {
return errors.New("ks apps with version 0.0.1 do not have support for targets")
}
// Upgrade upgrades the app to the latest apiVersion.
func (a *App001) Upgrade(dryRun bool) error {
if err := a.load(); err != nil {
return errors.Wrap(err, "load configuration")
}
if dryRun {
fmt.Fprintf(a.out, "\n[dry run] Upgrading application settings from version 0.0.1 to to 0.1.0.\n")
}
envs, err := a.Environments()