Commit dac3daa6 authored by Oren Shomron's avatar Oren Shomron
Browse files

Support prototype resolution from environment-scoped packages



Closes #713
Signed-off-by: default avatarOren Shomron <shomron@gmail.com>
parent 03566e32
......@@ -141,6 +141,7 @@ func (h *Helm) Prototypes() (prototype.Prototypes, error) {
APIVersion: prototype.DefaultAPIVersion,
Kind: prototype.DefaultKind,
Name: h.prototypeName(),
Version: latestVersion,
Template: prototype.SnippetSchema{
Description: shortDescription,
ShortDescription: shortDescription,
......
......@@ -128,6 +128,7 @@ func (l *Local) Prototypes() (prototype.Prototypes, error) {
if err != nil {
return err
}
spec.Version = l.version
prototypes = append(prototypes, spec)
return nil
......
......@@ -17,10 +17,12 @@ package prototype
import (
"fmt"
"sort"
"strconv"
"strings"
"github.com/blang/semver"
"github.com/ksonnet/ksonnet/pkg/util/version"
"github.com/pkg/errors"
)
......@@ -51,6 +53,7 @@ type Prototype struct {
Name string `json:"name"`
Params ParamSchemas `json:"params"`
Template SnippetSchema `json:"template"`
Version string `json:"-"` // Version of container package. Not serialized.
}
func (s *Prototype) validate() error {
......@@ -71,6 +74,22 @@ func (s *Prototype) validate() error {
// Prototypes is a slice of pointer to `SpecificationSchema`.
type Prototypes []*Prototype
// SortByVersion sorts a prototype list by package version.
func (p Prototypes) SortByVersion() {
less := func(i, j int) bool {
vI, err := version.Make(p[i].Version)
vJ, err2 := version.Make(p[j].Version)
if err != nil || err2 != nil {
// Fall back to lexical sort
return p[i].Version < p[j].Version
}
return vI.LT(vJ)
}
sort.Slice(p, less)
}
// RequiredParams retrieves all parameters that are required by a prototype.
func (s *Prototype) RequiredParams() ParamSchemas {
reqd := ParamSchemas{}
......
......@@ -17,6 +17,7 @@ package registry
import (
"fmt"
"strings"
"github.com/ksonnet/ksonnet/pkg/app"
"github.com/ksonnet/ksonnet/pkg/parts"
......@@ -54,16 +55,20 @@ type packageManager struct {
app app.App
InstallChecker pkg.InstallChecker
packagesFn func() ([]pkg.Package, error)
}
var _ PackageManager = (*packageManager)(nil)
// NewPackageManager creates an instance of PackageManager.
func NewPackageManager(a app.App) PackageManager {
return &packageManager{
pm := packageManager{
app: a,
InstallChecker: &pkg.DefaultInstallChecker{App: a},
}
pm.packagesFn = pm.Packages
return &pm
}
// Find finds a package by name. Package names have the format `<registry>/<library>@<version>`.
......@@ -184,12 +189,12 @@ func (m *packageManager) Packages() ([]pkg.Package, error) {
return nil, errors.New("app is required")
}
libraryConfigs, err := m.app.Libraries()
libIndex, err := allLibraries(m.app)
if err != nil {
return nil, errors.Wrap(err, "reading libraries defined in the configuration")
return nil, errors.Wrapf(err, "resolving libraries")
}
// TODO - Check libraries configured under environments?
libraryConfigs := uniqueLibsByVersion(libIndex)
registryConfigs, err := m.app.Registries()
if err != nil {
......@@ -198,14 +203,14 @@ func (m *packageManager) Packages() ([]pkg.Package, error) {
packages := make([]pkg.Package, 0)
for k, libraryConfig := range libraryConfigs {
for _, libraryConfig := range libraryConfigs {
registryConfig, ok := registryConfigs[libraryConfig.Registry]
if !ok {
return nil, errors.Errorf("registry %q required by library %q is not defined in the configuration",
libraryConfig.Registry, k)
libraryConfig.Registry, libraryConfig.Name)
}
p, err := m.loadPackage(registryConfig, k, libraryConfig.Registry, libraryConfig.Version)
p, err := m.loadPackage(registryConfig, libraryConfig.Name, libraryConfig.Registry, libraryConfig.Version)
if err != nil {
return nil, err
}
......@@ -290,31 +295,61 @@ func (m *packageManager) loadPackage(registryConfig *app.RegistryConfig, pkgName
}
func (m *packageManager) Prototypes() (prototype.Prototypes, error) {
packages, err := m.Packages()
packages, err := m.packagesFn()
if err != nil {
return nil, errors.Wrap(err, "loading packages")
}
var prototypes prototype.Prototypes
var result prototype.Prototypes
// Index prototypes by name
byName := make(map[string]prototype.Prototypes)
for _, p := range packages {
protos, err := p.Prototypes()
if err != nil {
return nil, errors.Wrap(err, "loading prototypes")
}
prototypes = append(prototypes, protos...)
for _, p := range protos {
lst := byName[p.Name]
lst = append(lst, p)
byName[p.Name] = lst
}
}
return prototypes, nil
for _, protos := range byName {
if len(protos) == 0 {
continue
}
p := latestPrototype(protos)
if p == nil {
continue
}
result = append(result, p)
}
return result, nil
}
type libraryByVersion map[string]*app.LibraryConfig
type librariesByVersion map[string][]*app.LibraryConfig
// Index libraries by descriptor, each can have multiple distinct versions.
// The same library can be indexed under multiple permutations of its fully-qualified descriptor
type libraryByDesc map[pkg.Descriptor]libraryByVersion
func (index libraryByDesc) String() string {
var sb strings.Builder
for d, byVersion := range index {
sb.WriteString(fmt.Sprintf("[%v]: byVersion=%v\n", d, byVersion))
}
return sb.String()
}
func indexLibrary(index libraryByDesc, d pkg.Descriptor, l *app.LibraryConfig) {
byVer, ok := index[d]
if !ok {
......@@ -322,7 +357,9 @@ func indexLibrary(index libraryByDesc, d pkg.Descriptor, l *app.LibraryConfig) {
index[d] = byVer
}
byVer[d.Version] = l
// NOTE d.Version is not always equal to l.Version - we index the same
// library under multiple descriptors to facilitate searching.
byVer[l.Version] = l
}
func indexLibraryPermutations(index libraryByDesc, l *app.LibraryConfig) {
......@@ -348,7 +385,7 @@ func indexLibraryPermutations(index libraryByDesc, l *app.LibraryConfig) {
// in search using partial keys.
func allLibraries(a app.App) (libraryByDesc, error) {
if a == nil {
return nil, errors.Errorf("nil receiver")
return nil, errors.Errorf("nil app")
}
index := libraryByDesc{}
......@@ -374,6 +411,40 @@ func allLibraries(a app.App) (libraryByDesc, error) {
return index, nil
}
// latestPrototype returns the latest prototype from the provided list.
// The list should represent different versions of the same prototype, as defined by having
// the same unqualified name. The list will not be modified.
func latestPrototype(protos prototype.Prototypes) *prototype.Prototype {
if len(protos) == 0 {
return nil
}
sorted := make(prototype.Prototypes, len(protos))
copy(sorted, protos)
sorted.SortByVersion()
return sorted[len(sorted)-1]
}
// Given an index of libaries (as created by allLibraries),
// return flag list of unique libraries, as distinguished by key registry:name:version.
func uniqueLibsByVersion(libIndex libraryByDesc) []*app.LibraryConfig {
var result = make([]*app.LibraryConfig, 0, len(libIndex))
for d, byVersion := range libIndex {
// Skip overly qualified indexes (remaining packages will be unique by version)
if d.Name == "" || d.Registry == "" || d.Version != "" {
continue
}
for _, l := range byVersion {
result = append(result, l)
}
}
return result
}
// IsInstalled determines whether the specified package is installed.
// Only Name is required in the descriptor, Registry and Version are inferred as "any"
// if missing. IsInstalled will make as specific a match as possible.
......
......@@ -22,6 +22,7 @@ import (
amocks "github.com/ksonnet/ksonnet/pkg/app/mocks"
"github.com/ksonnet/ksonnet/pkg/parts"
"github.com/ksonnet/ksonnet/pkg/pkg"
"github.com/ksonnet/ksonnet/pkg/prototype"
"github.com/ksonnet/ksonnet/pkg/util/test"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
......@@ -62,9 +63,16 @@ func Test_packageManager_Find(t *testing.T) {
func Test_packageManager_Packages(t *testing.T) {
test.WithApp(t, "/app", func(a *amocks.App, fs afero.Fs) {
makePkg := func(name, registry, version string) pkg.Package {
p, err := pkg.NewLocal(a, name, registry, version, &pkg.DefaultInstallChecker{App: a})
require.NoErrorf(t, err, "creating package %s/%s@%s", registry, name, version)
return p
}
test.StageDir(t, fs, "incubator/apache", "/work/apache")
test.StageDir(t, fs, "incubator/apache", "/app/vendor/incubator/apache")
test.StageDir(t, fs, "incubator/nginx", "/app/vendor/incubator/nginx@2.0.0")
test.StageDir(t, fs, "incubator/nginx", "/app/vendor/incubator/nginx@1.2.3")
a.On("VendorPath").Return("/app/vendor")
......@@ -80,57 +88,130 @@ func Test_packageManager_Packages(t *testing.T) {
libraries := app.LibraryConfigs{
"apache": &app.LibraryConfig{
Registry: "incubator",
Name: "apache",
},
"nginx": &app.LibraryConfig{
Registry: "incubator",
Name: "nginx",
Version: "2.0.0",
},
}
a.On("Libraries").Return(libraries, nil)
envLibraries := app.LibraryConfigs{
"nginx": &app.LibraryConfig{
Registry: "incubator",
Name: "nginx",
Version: "1.2.3",
},
}
environments := app.EnvironmentConfigs{
"default": &app.EnvironmentConfig{
Name: "default",
Libraries: envLibraries,
},
}
a.On("Environments").Return(environments, nil)
// Expect global libraries + envLibraries
expected := make([]pkg.Package, 0, len(libraries)+len(envLibraries))
for _, l := range libraries {
p := makePkg(l.Name, l.Registry, l.Version)
expected = append(expected, p)
}
for _, l := range envLibraries {
p := makePkg(l.Name, l.Registry, l.Version)
expected = append(expected, p)
}
pm := NewPackageManager(a)
packages, err := pm.Packages()
require.NoError(t, err)
require.Len(t, packages, 1)
p := packages[0]
require.Equal(t, "apache", p.Name())
assert.Len(t, packages, len(libraries)+len(envLibraries))
assert.Subset(t, expected, packages)
})
}
func Test_packageManager_Prototypes(t *testing.T) {
test.WithApp(t, "/app", func(a *amocks.App, fs afero.Fs) {
makePkg := func(name, registry, version string) pkg.Package {
p, err := pkg.NewLocal(a, name, registry, version, &pkg.DefaultInstallChecker{App: a})
require.NoErrorf(t, err, "creating package %s/%s@%s", registry, name, version)
return p
}
test.StageDir(t, fs, "incubator/apache", "/work/apache")
test.StageDir(t, fs, "incubator/apache", "/app/vendor/incubator/apache")
test.StageDir(t, fs, "incubator/apache", "/app/vendor/incubator/apache@1.2.3")
a.On("VendorPath").Return("/app/vendor")
registries := app.RegistryConfigs{
"incubator": &app.RegistryConfig{
Protocol: "fs",
URI: "/work",
},
pkgs := []pkg.Package{
makePkg("apache", "incubator", ""),
makePkg("apache", "incubator", "2.0.1"),
makePkg("apache", "incubator", "1.2.3"),
}
a.On("Registries").Return(registries, nil)
libraries := app.LibraryConfigs{
"apache": &app.LibraryConfig{
Registry: "incubator",
pm := packageManager{
app: a,
InstallChecker: &pkg.DefaultInstallChecker{App: a},
packagesFn: func() ([]pkg.Package, error) {
return pkgs, nil
},
}
a.On("Libraries").Return(libraries, nil)
pm := NewPackageManager(a)
protos, err := pm.Prototypes()
require.NoError(t, err)
// We expect the prototype to be retuned by only one of the packages
require.Len(t, protos, 1)
assert.Equal(t, "2.0.1", protos[0].Version)
})
}
func Test_latestPrototype(t *testing.T) {
protos := prototype.Prototypes{
&prototype.Prototype{
Version: "",
},
&prototype.Prototype{
Version: "2.4.5",
},
&prototype.Prototype{
Version: "v2.0.5",
},
&prototype.Prototype{
Version: "1.2.3",
},
}
p := latestPrototype(protos)
assert.Equal(t, "2.4.5", p.Version)
}
func Test_latestPrototype_Non_Semver(t *testing.T) {
protos := prototype.Prototypes{
&prototype.Prototype{
Version: "",
},
&prototype.Prototype{
Version: "semanticLargest",
},
&prototype.Prototype{
Version: "abcd",
},
&prototype.Prototype{
Version: "notsemver",
},
}
p := latestPrototype(protos)
assert.Equal(t, "semanticLargest", p.Version)
}
func Test_remotePackage(t *testing.T) {
rp := &remotePackage{
registryName: "registry-name",
......
// Copyright 2018 The ksonnet authors
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package version
import "sort"
......
// Copyright 2018 The ksonnet authors
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package version
import (
......
// Copyright 2018 The ksonnet authors
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package version
import (
......@@ -45,6 +60,11 @@ func Make(s string) (Version, error) {
}, nil
}
func (v *Version) String() string {
func (v Version) String() string {
return v.raw
}
// LT checks if v is less than o.
func (v Version) LT(o Version) bool {
return v.v.LT(o.v)
}
// Copyright 2018 The ksonnet authors
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package version
import (
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment