Skip to content

GitLab

Commit 3d8b3758 authored by Oren Shomron's avatar Oren Shomron
Browse files

Refactor schema to support explicit migrations 



* Dropped support for 0.0.1 apps
* Versioning has been pushed up into the Schema types instead of App.
* Added migrations framework for migrating schema versions, one hop at time

Closes #849
Signed-off-by: default avatarOren Shomron <shomron@gmail.com>
parent fa2429c7
Hide whitespace changes
Inline Side-by-side
Showing with 2329 additions and 2029 deletions
pkg/actions/actions.go
View file @ 3d8b3758
......@@ -16,6 +16,7 @@
package actions
import (
"bytes"
"crypto/tls"
"fmt"
"net"
......@@ -24,6 +25,8 @@ import (
"github.com/ksonnet/ksonnet/pkg/app"
"github.com/ksonnet/ksonnet/pkg/client"
"github.com/ksonnet/ksonnet/pkg/registry"
"github.com/ksonnet/ksonnet/pkg/upgrade"
"github.com/pkg/errors"
"github.com/spf13/afero"
)
......@@ -378,7 +381,8 @@ func (o *optionLoader) LoadApp() app.App {
}
if !o.LoadOptionalBool(OptionSkipCheckUpgrade) {
if _, err := a.CheckUpgrade(); err != nil {
pm := registry.NewPackageManager(a)
if _, err := upgrade.CheckUpgrade(a, new(bytes.Buffer), pm, false); err != nil {
o.err = errors.Wrap(err, "checking for app upgrades")
return nil
}
......
pkg/app/app.go
View file @ 3d8b3758
......@@ -22,7 +22,6 @@ import (
"sort"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/afero"
)
......@@ -69,8 +68,6 @@ type App interface {
Fs() afero.Fs
// HTTPClient is the app's http client
HTTPClient() *http.Client
// CheckUpgrade checks whether an app should be upgraded.
CheckUpgrade() (bool, error)
// LibPath returns the path of the lib for an environment.
LibPath(envName string) (string, error)
// Libraries returns all environments.
......@@ -95,42 +92,27 @@ type App interface {
UpdateLib(name string, env string, spec *LibraryConfig) (*LibraryConfig, error)
// UpdateRegistry updates a registry.
UpdateRegistry(spec *RegistryConfig) error
// Upgrade upgrades an application to the current version.
Upgrade(dryRun bool) error
// Upgrade upgrades an application (app.yaml) to the current version.
Upgrade(bool) error
// VendorPath returns the root of the vendor path.
VendorPath() string
}
// Load loads the application configuration.
func Load(fs afero.Fs, httpClient *http.Client, appRoot string) (App, error) {
log := log.WithField("action", "app.Load")
spec, err := read(fs, appRoot)
if os.IsNotExist(err) {
// During `ks init`, app.yaml will not yet exist - generate a new one.
return NewApp010(fs, appRoot, httpClient), nil
return NewBaseApp(fs, appRoot, httpClient), nil
}
if err != nil {
return nil, errors.Wrap(err, "reading app configuration")
}
switch spec.APIVersion {
default:
return nil, errors.Errorf("unknown apiVersion %q in %s", spec.APIVersion, appYamlName)
case "0.0.1":
return NewApp001(fs, appRoot, httpClient), nil
case "0.1.0", "0.2.0":
// TODO TODO
// 0.1.0 will auto-upgraded to 0.2.0. 0.1.0 is read-compatible with
// 0.2.0, but will be persisted back as 0.2.0. This behavior will be
// subsequently changed with new upgrade framework.
a := NewApp010(fs, appRoot, httpClient)
log.Debugf("Interpreting app version as latest (0.2.0)", a.baseApp)
a.config.APIVersion = "0.2.0"
a.baseApp.config.APIVersion = "0.2.0"
return a, nil
}
a := NewBaseApp(fs, appRoot, httpClient)
a.config = spec
return a, nil
}
func app010LibPath(root string) string {
......
pkg/app/app001.go deleted 100644 → 0
View file @ fa2429c7
// Copyright 2018 The kubecfg authors
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package app
import (
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"github.com/ghodss/yaml"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/afero"
)
const (
// app001specJSON is the name for environment schema
app001specJSON = "spec.json"
)
// App001 is a ksonnet 0.0.1 application.
type App001 struct {
out io.Writer
*baseApp
}
var _ App = (*App001)(nil)
// NewApp001 creates an App001 instance.
func NewApp001(fs afero.Fs, root string, httpClient *http.Client) *App001 {
ba := newBaseApp(fs, root, httpClient)
return &App001{
out: os.Stdout,
baseApp: ba,
}
}
// AddEnvironment adds an environment spec to the app spec. If the spec already exists,
// it is overwritten.
func (a *App001) AddEnvironment(e *EnvironmentConfig, k8sSpecFlag string, isOverride bool) error {
if e == nil {
return errors.Errorf("nil environment configuraion")
}
if e.Name == "" {
return errors.Errorf("invalid environment name")
}
// if it is an override, write the destination to override file. If not, do the normal thing.
envPath := filepath.Join(a.root, EnvironmentDirName, e.Name)
if err := a.fs.MkdirAll(envPath, DefaultFolderPermissions); err != nil {
return err
}
specPath := filepath.Join(envPath, app001specJSON)
b, err := json.Marshal(e.Destination)
if err != nil {
return err
}
if err = afero.WriteFile(a.fs, specPath, b, DefaultFilePermissions); err != nil {
return err
}
_, err = a.libUpdater.UpdateKSLib(k8sSpecFlag, a.appLibPath(e.Name))
return err
}
func (a *App001) overrideDestintation(name string, envSpec *EnvironmentConfig) error {
return nil
}
// Environment returns the spec for an environment. In 0.1.0, the file lives in
// /environments/name/spec.json.
func (a *App001) Environment(name string) (*EnvironmentConfig, error) {
path := filepath.Join(a.root, EnvironmentDirName, name, app001specJSON)
return read001EnvSpec(a.fs, name, path)
}
// Environments returns specs for all environments. In 0.1.0, the environment spec
// lives in spec.json files.
func (a *App001) Environments() (EnvironmentConfigs, error) {
specs := EnvironmentConfigs{}
root := filepath.Join(a.root, EnvironmentDirName)
err := afero.Walk(a.fs, root, func(path string, fi os.FileInfo, err error) error {
if fi.IsDir() {
return nil
}
if fi.Name() == app001specJSON {
dir := filepath.Dir(path)
envName := strings.TrimPrefix(dir, root+"/")
spec, err := read001EnvSpec(a.fs, envName, path)
if err != nil {
return err
}
specs[envName] = spec
}
return nil
})
if err != nil {
return nil, err
}
return specs, nil
}
// CheckUpgrade initializes the App.
func (a *App001) CheckUpgrade() (bool, error) {
msg := "Your application's apiVersion is below 0.1.0. In order to use all ks features, you " +
"can upgrade your application using `ks upgrade`."
log.Warn(msg)
return true, nil
}
// LibPath returns the lib path for an env environment.
func (a *App001) LibPath(envName string) (string, error) {
return filepath.Join(a.envDir(envName), ".metadata"), nil
}
// Libraries returns application libraries.
func (a *App001) Libraries() (LibraryConfigs, error) {
if err := a.load(); err != nil {
return nil, errors.Wrap(err, "load configuration")
}
return a.config.Libraries, nil
}
// Registries returns application registries.
func (a *App001) Registries() (RegistryConfigs, error) {
if err := a.load(); err != nil {
return nil, errors.Wrap(err, "load configuration")
}
registries := RegistryConfigs{}
for k, v := range a.config.Registries {
registries[k] = v
}
for k, v := range a.overrides.Registries {
registries[k] = v
}
return registries, nil
}
// RemoveEnvironment removes an environment.
func (a *App001) RemoveEnvironment(envName string, override bool) error {
a.Fs().RemoveAll(a.envDir(envName))
return nil
}
// RenameEnvironment renames environments.
func (a *App001) RenameEnvironment(from, to string, override bool) error {
return moveEnvironment(a.fs, a.root, from, to)
}
// UpdateTargets returns an error since 0.0.1 based applications don't have support
// for targets.
func (a *App001) UpdateTargets(envName string, targets []string) error {
return errors.New("ks apps with version 0.0.1 do not have support for targets")
}
// Upgrade upgrades the app to the latest apiVersion.
func (a *App001) Upgrade(dryRun bool) error {
if err := a.load(); err != nil {
return errors.Wrap(err, "load configuration")
}
if dryRun {
fmt.Fprintf(a.out, "\n[dry run] Upgrading application settings from version 0.0.1 to to 0.1.0.\n")
}
envs, err := a.Environments()
if err != nil {
return err
}
if dryRun {
fmt.Fprintf(a.out, "[dry run] Converting 0.0.1 environments to 0.1.0a:\n")
}
for _, env := range envs {
a.convertEnvironment(env.Path, dryRun)
}
a.config.APIVersion = "0.1.0"
if dryRun {
data, err := yaml.Marshal(a.config)
if err != nil {
return err
}
fmt.Fprintf(a.out, "\n[dry run] Upgraded app.yaml:\n%s\n", string(data))
fmt.Fprintf(a.out, "[dry run] You can perform the migration by running `ks upgrade`.\n")
return nil
}
return a.save()
}
type k8sSchema struct {
Info struct {
Version string `json:"version,omitempty"`
} `json:"info,omitempty"`
}
func read001EnvSpec(fs afero.Fs, name, path string) (*EnvironmentConfig, error) {
b, err := afero.ReadFile(fs, path)
if err != nil {
return nil, err
}
var s EnvironmentDestinationSpec
if err = json.Unmarshal(b, &s); err != nil {
return nil, err
}
if s.Namespace == "" {
s.Namespace = "default"
}
envPath := filepath.Dir(path)
swaggerPath := filepath.Join(envPath, ".metadata", "swagger.json")
b, err = afero.ReadFile(fs, swaggerPath)
if err != nil {
return nil, err
}
var ks k8sSchema
if err = json.Unmarshal(b, &ks); err != nil {
return nil, err
}
if ks.Info.Version == "" {
return nil, errors.New("unable to determine environment Kubernetes version")
}
spec := EnvironmentConfig{
Path: name,
Destination: &s,
KubernetesVersion: ks.Info.Version,
}
return &spec, nil
}
func (a *App001) convertEnvironment(envName string, dryRun bool) error {
if err := a.load(); err != nil {
return errors.Wrap(err, "load configuration")
}
path := filepath.Join(a.root, EnvironmentDirName, envName, "spec.json")
env, err := read001EnvSpec(a.fs, envName, path)
if err != nil {
return err
}
a.config.Environments[envName] = env
if dryRun {
fmt.Fprintf(a.out, "[dry run]\t* adding the environment description in environment `%s to `app.yaml`.\n",
envName)
return nil
}
if err = a.fs.Remove(path); err != nil {
return err
}
k8sSpecFlag := fmt.Sprintf("version:%s", env.KubernetesVersion)
_, err = a.libUpdater.UpdateKSLib(k8sSpecFlag, app010LibPath(a.root))
if err != nil {
return err
}
return a.save()
}
func (a *App001) appLibPath(envName string) string {
return filepath.Join(a.root, EnvironmentDirName, envName, ".metadata")
}
func (a *App001) envDir(envName string) string {
envParts := strings.Split(envName, "/")
envRoot := filepath.Join(a.Root(), EnvironmentDirName)
return filepath.Join(append([]string{envRoot}, envParts...)...)
}
pkg/app/app001_test.go deleted 100644 → 0
View file @ fa2429c7
// Copyright 2018 The kubecfg authors
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package app
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestApp001_RenameEnvironment(t *testing.T) {
cases := []struct {
name string
from string
to string
shouldExist []string
shouldNotExist []string
}{
{
name: "rename",
from: "default",
to: "renamed",
shouldExist: []string{
"/environments/renamed/.metadata",
"/environments/renamed/spec.json",
},
shouldNotExist: []string{
"/environments/default",
},
},
{
name: "rename to nested",
from: "default",
to: "default/nested",
shouldExist: []string{
"/environments/default/nested/.metadata",
"/environments/default/nested/spec.json",
},
shouldNotExist: []string{
"/environments/default/.metadata",
},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
withApp001Fs(t, "app001_app.yaml", func(app *App001) {
err := app.RenameEnvironment(tc.from, tc.to, false)
require.NoError(t, err)
for _, p := range tc.shouldExist {
checkExist(t, app.Fs(), p)
}
for _, p := range tc.shouldNotExist {
checkNotExist(t, app.Fs(), p)
}
app.load()
require.NoError(t, err)
_, err = app.Environment(tc.from)
assert.Error(t, err)
_, err = app.Environment(tc.to)
assert.NoError(t, err)
})
})
}
}
func TestApp001_Environments(t *testing.T) {
withApp001Fs(t, "app001_app.yaml", func(app *App001) {
expected := EnvironmentConfigs{
"default": &EnvironmentConfig{
Destination: &EnvironmentDestinationSpec{
Namespace: "some-namespace",
Server: "http://example.com",
},
KubernetesVersion: "v1.7.0",
Path: "default",
},
"us-east/test": &EnvironmentConfig{
Destination: &EnvironmentDestinationSpec{
Namespace: "some-namespace",
Server: "http://example.com",
},
KubernetesVersion: "v1.7.0",
Path: "us-east/test",
},
"us-west/test": &EnvironmentConfig{
Destination: &EnvironmentDestinationSpec{
Namespace: "some-namespace",
Server: "http://example.com",
},
KubernetesVersion: "v1.7.0",
Path: "us-west/test",
},
"us-west/prod": &EnvironmentConfig{
Destination: &EnvironmentDestinationSpec{
Namespace: "some-namespace",
Server: "http://example.com",
},
KubernetesVersion: "v1.7.0",
Path: "us-west/prod",
},
}
envs, err := app.Environments()
require.NoError(t, err)
require.Equal(t, expected, envs)
})
}
func TestApp001_Environment(t *testing.T) {
cases := []struct {
name string
envName string
isErr bool
}{
{
name: "existing env",
envName: "us-east/test",
},
{
name: "invalid env",
envName: "missing",
isErr: true,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
withApp001Fs(t, "app001_app.yaml", func(app *App001) {
spec, err := app.Environment(tc.envName)
if tc.isErr {
require.Error(t, err)
} else {
require.NoError(t, err)
require.Equal(t, tc.envName, spec.Path)
}
})
})
}
}
func TestApp001_AddEnvironment(t *testing.T) {
withApp001Fs(t, "app001_app.yaml", func(app *App001) {
newEnv := &EnvironmentConfig{
Name: "us-west/qa",
Destination: &EnvironmentDestinationSpec{
Namespace: "some-namespace",
Server: "http://example.com",
},
Path: "us-west/qa",
}
k8sSpecFlag := "version:v1.8.7"
err := app.AddEnvironment(newEnv, k8sSpecFlag, false)
require.NoError(t, err)
_, err = app.Environment("us-west/qa")
require.NoError(t, err)
})
}
func TestApp001_Upgrade_dryrun(t *testing.T) {
withApp001Fs(t, "app001_app.yaml", func(app *App001) {
app.out = ioutil.Discard
err := app.Upgrade(true)
require.NoError(t, err)
})
}
func TestApp001_Upgrade(t *testing.T) {
withApp001Fs(t, "app001_app.yaml", func(app *App001) {
app.out = ioutil.Discard
err := app.Upgrade(false)
require.NoError(t, err)
root := filepath.Join(app.Root(), EnvironmentDirName)
var foundSpec bool
err = afero.Walk(app.Fs(), root, func(path string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
if fi.IsDir() {
return nil