Commit ada571f6 authored by Oren Shomron's avatar Oren Shomron
Browse files

Add versioned, environment-scoped package support



This change adds version-mapped package support to environments. An environment can specify the specific version of a package to consume - and multiple, side-by-side versions of a package can be installed in the package cache.

Closes #631
Closes #651

* Deprecate GitVersion from LibraryConfig
* pkg.Descriptor.Part -> pkg.Descriptor.Name
* Add Version, Path to PackageManager.
* Composed package structs
* Revendoring - Tailor import path to environment's packages, allow version-free import strings
* Skip unversioned packages when revendoring
* Allow injection of custom importers into jsonnet.VM wrapper
* Allow passing VMOpts to VM via Evaluate*
* Add versioned package evaluation test
* Allow versioned packages to fall back to unversioned paths
* Fix DefaultInstallChecker shadowed variable when looking up environment packages
* Test skipping of missing paths in revendorPackages
* Tweak CacheDependency->ResolveLibrary interface - onFile paths should always be relative to the registry root
* Ensure 0.2.0 version is output when re-writing app.yaml
Signed-off-by: default avatarOren Shomron <shomron@gmail.com>
parent 842d77c0
......@@ -72,14 +72,14 @@ func (pi *PkgInstall) Run() error {
}
func (pi *PkgInstall) parseDepSpec() (pkg.Descriptor, string, error) {
d, err := pkg.ParseName(pi.libName)
d, err := pkg.Parse(pi.libName)
if err != nil {
return pkg.Descriptor{}, "", err
}
customName := pi.customName
if customName == "" {
customName = d.Part
customName = d.Name
}
return d, customName, nil
......
......@@ -33,7 +33,7 @@ func TestPkgInstall(t *testing.T) {
dc := func(a app.App, d pkg.Descriptor, cn string) error {
expectedD := pkg.Descriptor{
Registry: "incubator",
Part: "apache",
Name: "apache",
}
require.Equal(t, expectedD, d)
require.Equal(t, "customName", cn)
......
......@@ -3,3 +3,4 @@ kubernetesversion: v1.7.0
path: ""
destination: null
targets: []
libraries: {}
......@@ -22,6 +22,7 @@ import (
"github.com/ksonnet/ksonnet/pkg/lib"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/afero"
)
......@@ -100,6 +101,7 @@ type App interface {
// Load loads the application configuration.
func Load(fs afero.Fs, cwd string, skipFindRoot bool) (App, error) {
log := log.WithField("action", "app.Load")
appRoot := cwd
if !skipFindRoot {
var err error
......@@ -109,18 +111,32 @@ func Load(fs afero.Fs, cwd string, skipFindRoot bool) (App, error) {
}
}
log.Debugf("called")
spec, err := read(fs, appRoot)
if err != nil {
if os.IsNotExist(err) {
// During `ks init`, app.yaml will not yet exist - generate a new one.
return NewApp010(fs, appRoot), nil
}
if err != nil {
return nil, errors.Wrap(err, "reading app configuration")
}
switch spec.APIVersion {
default:
return nil, errors.Errorf("unknown apiVersion %q in %s", spec.APIVersion, appYamlName)
case "0.0.1":
return NewApp001(fs, appRoot), nil
case "0.1.0":
return NewApp010(fs, appRoot), nil
case "0.1.0", "0.2.0":
// TODO TODO
// 0.1.0 will auto-upgraded to 0.2.0. 0.1.0 is read-compatible with
// 0.2.0, but will be persisted back as 0.2.0. This behavior will be
// subsequently changed with new upgrade framework.
a := NewApp010(fs, appRoot)
log.Debugf("Upgrading app [%p] version to latest (0.2.0)", a.baseApp)
a.config.APIVersion = "0.2.0"
a.baseApp.config.APIVersion = "0.2.0"
return a, nil
}
}
......
......@@ -209,7 +209,7 @@ func (a *App001) Upgrade(dryRun bool) error {
}
fmt.Fprintf(a.out, "\n[dry run] Upgraded app.yaml:\n%s\n", string(data))
fmt.Fprintf(a.out, "[dry run] You can preform the migration by running `ks upgrade`.\n")
fmt.Fprintf(a.out, "[dry run] You can perform the migration by running `ks upgrade`.\n")
return nil
}
......
......@@ -139,7 +139,7 @@ func (a *App010) Init() error {
}
msg := "Your application's apiVersion is 0.1.0, but legacy environment declarations " +
"where found in environments: %s. In order to proceed, you will have to run `ks upgrade` to " +
"were found in environments: %s. In order to proceed, you will have to run `ks upgrade` to " +
"upgrade your application. <see url>"
return errors.Errorf(msg, strings.Join(legacyEnvs, ", "))
......
......@@ -21,6 +21,7 @@ import (
"github.com/ghodss/yaml"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/afero"
)
......@@ -70,9 +71,20 @@ func (ba *baseApp) overridePath() string {
}
func (ba *baseApp) save() error {
log := log.WithField("action", "baseApp.save")
log.Debugf("called [%p]", ba)
ba.mu.Lock()
defer ba.mu.Unlock()
if ba.config == nil {
return errors.Errorf("cannot save nil app configuration")
}
// Signal we have converted to new app version
ba.config.APIVersion = DefaultAPIVersion
log.Debugf("saving app version %v", ba.config.APIVersion)
configData, err := yaml.Marshal(ba.config)
if err != nil {
return errors.Wrap(err, "convert application configuration to YAML")
......@@ -94,6 +106,9 @@ func (ba *baseApp) save() error {
}
func (ba *baseApp) load() error {
log := log.WithField("action", "baseApp.load")
log.Debugf("called [%p]", ba)
ba.mu.Lock()
defer ba.mu.Unlock()
......
......@@ -103,7 +103,7 @@ func Test_baseApp_AddRegistry_override(t *testing.T) {
err := ba.AddRegistry(reg, true)
require.NoError(t, err)
assertContents(t, fs, "app010_app.yaml", ba.configPath())
assertContents(t, fs, "app020_app.yaml", ba.configPath())
assertContents(t, fs, "add-registry-override.yaml", ba.overridePath())
}
......
......@@ -26,7 +26,7 @@ const (
// overrideKind is the override resource type.
overrideKind = "ksonnet.io/app-override"
// overrideVersion is the version of the override resource.
overrideVersion = "0.1.0"
overrideVersion = "0.2.0"
)
// Override defines overrides to ksonnet project configurations.
......@@ -52,7 +52,7 @@ func (o *Override) Validate() error {
// IsDefined returns true if the override has environments or registries defined.
func (o *Override) IsDefined() bool {
return len(o.Environments) > 0 || len(o.Registries) > 0
return o != nil && (len(o.Environments) > 0 || len(o.Registries) > 0)
}
// SaveOverride saves the override to the filesystem.
......
......@@ -29,7 +29,7 @@ import (
const (
// DefaultAPIVersion is the default ks API version to use if not specified.
DefaultAPIVersion = "0.1.0"
DefaultAPIVersion = "0.2.0"
// Kind is the schema resource type.
Kind = "ksonnet.io/app"
// DefaultVersion is the default version of the app schema.
......@@ -49,6 +49,21 @@ var (
ErrEnvironmentNotExists = fmt.Errorf("Environment with name doesn't exist")
)
var (
compatibleAPIRangeStrings = []string{
">= 0.0.1 <= 0.2.0",
}
compatibleAPIRanges = mustCompileRanges()
)
func mustCompileRanges() []semver.Range {
result := make([]semver.Range, 0, len(compatibleAPIRangeStrings))
for _, s := range compatibleAPIRangeStrings {
result = append(result, semver.MustParseRange(s))
}
return result
}
// Spec defines all the ksonnet project metadata. This includes details such as
// the project name, authors, environments, and registries.
type Spec struct {
......@@ -240,11 +255,16 @@ func (r *RegistryConfigs) UnmarshalJSON(b []byte) error {
}
// Set Name fields according to map keys
result := RegistryConfigs{}
for k, v := range registries {
if v == nil {
continue
}
v.Name = k
result[k] = v
}
*r = RegistryConfigs(registries)
*r = result
return nil
}
......@@ -261,11 +281,16 @@ func (e *EnvironmentConfigs) UnmarshalJSON(b []byte) error {
}
// Set Name fields according to map keys
result := EnvironmentConfigs{}
for k, v := range envs {
if v == nil {
continue
}
v.Name = k
result[k] = v
}
*e = EnvironmentConfigs(envs)
*e = result
return nil
}
......@@ -284,6 +309,8 @@ type EnvironmentConfig struct {
// Targets contain the relative component paths that this environment
// wishes to deploy on it's destination.
Targets []string `json:"targets,omitempty"`
// Libraries specifies versioned libraries specifically used by this environment.
Libraries LibraryConfigs `json:"libraries,omitempty"`
isOverride bool
}
......@@ -313,6 +340,13 @@ type EnvironmentDestinationSpec struct {
// LibraryConfig is the specification for a library part.
type LibraryConfig struct {
Name string `json:"name"`
Registry string `json:"registry"`
Version string `json:"version"`
}
// 0.1.0 version of LibraryConfig
type libraryConfigDeprecated struct {
Name string `json:"name"`
Registry string `json:"registry"`
Version string `json:"version"`
......@@ -328,6 +362,61 @@ type GitVersionSpec struct {
// LibraryConfigs is a mapping of a library configurations by name.
type LibraryConfigs map[string]*LibraryConfig
// UnmarshalJSON implements the json.Unmarshaler interface.
// We implement some compatibility conversions.
func (l *LibraryConfigs) UnmarshalJSON(b []byte) error {
var cfgs map[string]*LibraryConfig
if err := json.Unmarshal(b, &cfgs); err != nil {
return err
}
result := LibraryConfigs{}
for k, v := range cfgs {
if v == nil {
continue
}
v.Name = k
result[k] = v
}
*l = result
return nil
}
// libraryConfig is an alias that allows us to leverage default JSON decoding
// in our custom UnmarshalJSON handler without triggering infinite recursion.
type libraryConfig LibraryConfig
// UnmarshalJSON implements the json.Unmarshaler interface.
// We implement some compatibility conversions.
func (l *LibraryConfig) UnmarshalJSON(b []byte) error {
var cfg libraryConfig
if err := json.Unmarshal(b, &cfg); err != nil {
return err
}
*l = LibraryConfig(cfg)
// Check if there's any need for conversions
if cfg.Version != "" {
return nil
}
// Try to convert deprecated fields
var oldStyle libraryConfigDeprecated
if err := json.Unmarshal(b, &oldStyle); err != nil {
// This is best-effort, not an error
return nil
}
if oldStyle.GitVersion != nil {
l.Version = oldStyle.GitVersion.CommitSHA
}
return nil
}
// ContributorSpec is a specification for the project contributors.
type ContributorSpec struct {
Name string `json:"name"`
......@@ -390,17 +479,23 @@ func (s *Spec) validate() error {
return errors.New("invalid version")
}
compatVer, _ := semver.Make(DefaultAPIVersion)
ver, err := semver.Make(s.APIVersion)
if err != nil {
return errors.Wrap(err, "Failed to parse version in app spec")
}
if compatVer.Compare(ver) < 0 {
var compatible bool
for _, compatRange := range compatibleAPIRanges {
if compatRange(ver) {
compatible = true
}
}
if !compatible {
return fmt.Errorf(
"Current app uses unsupported spec version '%s' (this client only supports %s)",
s.APIVersion,
DefaultAPIVersion)
compatibleAPIRangeStrings)
}
return nil
......
......@@ -188,7 +188,7 @@ func TestGetEnvironmentConfigs(t *testing.T) {
// Verifies that EnvironmentConfig.Name fields are injected at unmarshal time.
func TestEnvironmentConfigHasName(t *testing.T) {
b := []byte(`
apiVersion: 0.1.0
apiVersion: 0.2.0
environments:
default:
destination:
......@@ -382,7 +382,7 @@ func Test_write(t *testing.T) {
fs := afero.NewMemMapFs()
spec := &Spec{
APIVersion: "0.1.0",
APIVersion: "0.2.0",
Environments: EnvironmentConfigs{
"a": &EnvironmentConfig{},
"b": &EnvironmentConfig{isOverride: true},
......@@ -407,7 +407,7 @@ func Test_write_no_override(t *testing.T) {
fs := afero.NewMemMapFs()
spec := &Spec{
APIVersion: "0.1.0",
APIVersion: "0.2.0",
Environments: EnvironmentConfigs{
"a": &EnvironmentConfig{},
},
......@@ -435,7 +435,7 @@ func Test_read(t *testing.T) {
require.NoError(t, err)
expected := &Spec{
APIVersion: "0.1.0",
APIVersion: "0.2.0",
Contributors: ContributorSpecs{},
Environments: EnvironmentConfigs{
"a": &EnvironmentConfig{Name: "a"},
......@@ -463,9 +463,17 @@ func TestEnvironmentSpec_MakePath(t *testing.T) {
}
// Test that RegistryConfigs are properly deserialized, specifically
// their Name fields, which are handler by customer UnmarshalJSON code.
// their Name fields, which are handler by custom UnmarshalJSON code.
func TestUnmarshalRegistryConfigs(t *testing.T) {
input := []byte(`
tests := []struct {
name string
input string
expectedRegistries int
}{
{
name: "0.1.0",
expectedRegistries: 3,
input: `
apiVersion: 0.1.0
registries:
incubator:
......@@ -481,15 +489,38 @@ registries:
protocol: github
uri: github.com/ksonnet/parts/tree/next/incubator
version: 0.0.1
`)
var spec Spec
`,
},
{
name: "0.2.0",
expectedRegistries: 3,
input: `
apiVersion: 0.2.0
registries:
incubator:
protocol: github
uri: github.com/ksonnet/parts/tree/master/incubator
helm-stable:
protocol: helm
uri: https://kubernetes-charts.storage.googleapis.com
otherRegistry:
protocol: github
uri: github.com/ksonnet/parts/tree/next/incubator
version: 0.0.1
`,
},
}
err := yaml.Unmarshal(input, &spec)
assert.NoError(t, err)
assert.Equal(t, 3, len(spec.Registries))
for _, tc := range tests {
var spec Spec
for k, v := range spec.Registries {
assert.Equal(t, k, v.Name)
err := yaml.Unmarshal([]byte(tc.input), &spec)
assert.NoError(t, err, tc.name)
assert.Equal(t, tc.expectedRegistries, len(spec.Registries), tc.name)
for k, v := range spec.Registries {
assert.Equal(t, k, v.Name, tc.name)
}
}
}
......
apiVersion: 0.1.0
apiVersion: 0.2.0
kind: ksonnet.io/app-override
registries:
new:
......
apiVersion: 0.1.0
apiVersion: 0.2.0
environments:
default:
destination:
......
apiVersion: 0.2.0
environments:
default:
destination:
namespace: some-namespace
server: http://example.com
k8sVersion: v1.7.0
path: default
us-east/test:
destination:
namespace: some-namespace
server: http://example.com
k8sVersion: v1.7.0
path: us-east/test
us-west/prod:
destination:
namespace: some-namespace
server: http://example.com
k8sVersion: v1.7.0
path: us-west/prod
us-west/test:
destination:
namespace: some-namespace
server: http://example.com
k8sVersion: v1.7.0
path: us-west/test
kind: ksonnet.io/app
name: test-get-envs
registries:
incubator:
protocol: ""
uri: ""
version: 0.0.1
apiVersion: 0.1.0
apiVersion: 0.2.0
environments:
a:
destination: null
......
apiVersion: 0.1.0
apiVersion: 0.2.0
environments:
b:
destination: null
......
......@@ -73,7 +73,7 @@ ks apply dev --dry-run
ks apply dev -c guestbook-ui
# Create or update multiple components in a ksonnet application (e.g. 'guestbook-ui'
# and 'ngin-depl') for the 'dev' environment. Does not create resources that are
# and 'nginx-depl') for the 'dev' environment. Does not create resources that are
# not already present on the cluster.
#
# This essentially deploys 'components/guestbook-ui.jsonnet' and
......
// Copyright 2018 The ksonnet authors
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cluster
import (
......
// Copyright 2018 The ksonnet authors
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cluster
import (
......
// Copyright 2018 The ksonnet authors
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cluster
import (
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment