Commit a6d9a034 authored by Oren Shomron's avatar Oren Shomron
Browse files

List environment-scoped packages in addition to globally-scoped packages



Part of #626

* Add RemotePackages method to package manager
* pkg list uses PackageManger rather than App
* Packages from GitHub registry inherit version from the registry,
  rather than that specified in registry.yaml.
* Rename registry.LibaryConfig -> LibraryConfig
Signed-off-by: default avatarOren Shomron <shomron@gmail.com>
parent d225af81
......@@ -22,6 +22,7 @@ import (
"strings"
"github.com/ksonnet/ksonnet/pkg/app"
"github.com/ksonnet/ksonnet/pkg/pkg"
"github.com/ksonnet/ksonnet/pkg/registry"
"github.com/ksonnet/ksonnet/pkg/util/table"
"github.com/pkg/errors"
......@@ -45,6 +46,7 @@ func RunPkgList(m map[string]interface{}) error {
// PkgList lists available registries
type PkgList struct {
app app.App
pm registry.PackageManager
onlyInstalled bool
outputType string
......@@ -56,8 +58,10 @@ type PkgList struct {
func NewPkgList(m map[string]interface{}) (*PkgList, error) {
ol := newOptionLoader(m)
app := ol.LoadApp()
rl := &PkgList{
app: ol.LoadApp(),
app: app,
pm: registry.NewPackageManager(app),
onlyInstalled: ol.LoadBool(OptionInstalled),
outputType: ol.LoadOptionalString(OptionOutput),
......@@ -74,38 +78,45 @@ func NewPkgList(m map[string]interface{}) (*PkgList, error) {
// Run runs the env list action.
func (pl *PkgList) Run() error {
registries, err := pl.registryListFn(pl.app)
pkgs, err := pl.pm.Packages()
if err != nil {
return err
}
var rows [][]string
appLibraries, err := pl.app.Libraries()
if err != nil {
return err
index := make(map[string]pkg.Package)
for _, p := range pkgs {
index[p.String()] = p
}
for _, r := range registries {
spec, err := r.FetchRegistrySpec()
if !pl.onlyInstalled {
// Merge in remote packages
remote, err := pl.pm.RemotePackages()
if err != nil {
return err
return errors.Wrap(err, "listing remote packages")
}
for libName, config := range spec.Libraries {
_, isInstalled := appLibraries[libName]
if pl.onlyInstalled && !isInstalled {
for _, p := range remote {
if _, ok := index[p.String()]; ok {
continue
}
rows = append(rows, pl.addRow(r.Name(), libName, config.Version, isInstalled))
index[p.String()] = p
}
}
// Build output
var rows [][]string
for _, p := range index {
isInstalled, err := p.IsInstalled()
if err != nil {
return err
}
rows = append(rows, pl.addRow(p.RegistryName(), p.Name(), p.Version(), isInstalled))
}
sort.Slice(rows, func(i, j int) bool {
nameI := strings.Join([]string{rows[i][0], rows[i][1]}, "-")
nameJ := strings.Join([]string{rows[j][0], rows[j][1]}, "-")
nameI := strings.Join([]string{rows[i][0], rows[i][1], rows[i][2]}, "-")
nameJ := strings.Join([]string{rows[j][0], rows[j][1], rows[j][2]}, "-")
return nameI < nameJ
})
......
......@@ -17,31 +17,82 @@ package actions
import (
"bytes"
"fmt"
"testing"
"github.com/ksonnet/ksonnet/pkg/app"
amocks "github.com/ksonnet/ksonnet/pkg/app/mocks"
"github.com/ksonnet/ksonnet/pkg/registry"
"github.com/ksonnet/ksonnet/pkg/pkg"
"github.com/ksonnet/ksonnet/pkg/prototype"
rmocks "github.com/ksonnet/ksonnet/pkg/registry/mocks"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
)
func TestPkgList(t *testing.T) {
withApp(t, func(appMock *amocks.App) {
libaries := app.LibraryConfigs{
"lib1": &app.LibraryConfig{},
}
type myPkg struct {
name string
registry string
version string
isInstalled bool
}
func (p myPkg) Name() string {
return p.name
}
func (p myPkg) Version() string {
return p.version
}
func (p myPkg) RegistryName() string {
return p.registry
}
func (p myPkg) Description() string {
return ""
}
func (p myPkg) Path() string {
return p.name
}
func (p myPkg) String() string {
return fmt.Sprintf("%s/%s@%s", p.registry, p.name, p.version)
}
func (p myPkg) Prototypes() (prototype.Prototypes, error) {
return nil, errors.New("not implemented")
}
func (p myPkg) IsInstalled() (bool, error) {
return p.isInstalled, nil
}
appMock.On("Libraries").Return(libaries, nil)
var _ = pkg.Package(myPkg{})
spec := &registry.Spec{
Libraries: registry.LibraryConfigs{
"lib1": &registry.LibaryConfig{Version: "0.0.1"},
"lib2": &registry.LibaryConfig{Version: "master"},
func TestPkgList(t *testing.T) {
withApp(t, func(appMock *amocks.App) {
pmMock := rmocks.PackageManager{}
pmMock.On("Packages").Return(
[]pkg.Package{
myPkg{
name: "lib1",
version: "0.0.1",
registry: "incubator",
isInstalled: true,
},
myPkg{
name: "lib1",
version: "0.0.2",
registry: "incubator",
isInstalled: true,
},
},
}
incubator := mockRegistry("incubator", false)
incubator.On("FetchRegistrySpec").Return(spec, nil)
nil,
)
pmMock.On("RemotePackages").Return(
[]pkg.Package{
myPkg{
name: "lib2",
version: "master",
registry: "incubator",
isInstalled: false,
},
},
nil,
)
cases := []struct {
name string
......@@ -81,11 +132,7 @@ func TestPkgList(t *testing.T) {
a, err := NewPkgList(in)
require.NoError(t, err)
a.registryListFn = func(app.App) ([]registry.Registry, error) {
registries := []registry.Registry{incubator}
return registries, nil
}
a.pm = &pmMock
var buf bytes.Buffer
a.out = &buf
......
......@@ -43,17 +43,17 @@ func TestRegistryDescribe(t *testing.T) {
spec := &registry.Spec{
Libraries: registry.LibraryConfigs{
"apache": &registry.LibaryConfig{Path: "apache"},
"efk": &registry.LibaryConfig{Path: "efk"},
"mariadb": &registry.LibaryConfig{Path: "mariadb"},
"memcached": &registry.LibaryConfig{Path: "memcached"},
"mongodb": &registry.LibaryConfig{Path: "mongodb"},
"mysql": &registry.LibaryConfig{Path: "mysql"},
"nginx": &registry.LibaryConfig{Path: "nginx"},
"node": &registry.LibaryConfig{Path: "node"},
"postres": &registry.LibaryConfig{Path: "postgres"},
"redis": &registry.LibaryConfig{Path: "redis"},
"tomcat": &registry.LibaryConfig{Path: "tomcat"},
"apache": &registry.LibraryConfig{Path: "apache"},
"efk": &registry.LibraryConfig{Path: "efk"},
"mariadb": &registry.LibraryConfig{Path: "mariadb"},
"memcached": &registry.LibraryConfig{Path: "memcached"},
"mongodb": &registry.LibraryConfig{Path: "mongodb"},
"mysql": &registry.LibraryConfig{Path: "mysql"},
"nginx": &registry.LibraryConfig{Path: "nginx"},
"node": &registry.LibraryConfig{Path: "node"},
"postres": &registry.LibraryConfig{Path: "postgres"},
"redis": &registry.LibraryConfig{Path: "redis"},
"tomcat": &registry.LibraryConfig{Path: "tomcat"},
},
}
......
REGISTRY NAME VERSION INSTALLED
======== ==== ======= =========
incubator lib1 0.0.1 *
incubator lib1 0.0.2 *
......@@ -7,6 +7,12 @@
"registry": "incubator",
"version": "0.0.1"
},
{
"installed": "*",
"name": "lib1",
"registry": "incubator",
"version": "0.0.2"
},
{
"installed": "",
"name": "lib2",
......
REGISTRY NAME VERSION INSTALLED
======== ==== ======= =========
incubator lib1 0.0.1 *
incubator lib1 0.0.2 *
incubator lib2 master
......@@ -74,6 +74,7 @@ func NewLocal(a app.App, name, registryName string, version string, installCheck
if err != nil {
return nil, errors.Wrap(err, "unmarshalling package configuration")
}
config.Version = version
return &Local{
pkg: pkg{
......
......@@ -76,6 +76,7 @@ func (p *pkg) String() string {
return "nil"
}
// TODO exclude @ if verion is empty
return fmt.Sprintf("%v/%v@%v", p.registryName, p.name, p.version)
}
......@@ -121,6 +122,15 @@ func (ic *DefaultInstallChecker) IsInstalled(name string) (bool, error) {
return isInstalled, nil
}
// TrueInstallChecker implements an always-true InstallChecker.
type TrueInstallChecker struct{}
// IsInstalled always returns true, signaling we knew the package was installed when it was
// bound to this installChecker.
func (ic TrueInstallChecker) IsInstalled(name string) (bool, error) {
return true, nil
}
// Package is a ksonnet package.
type Package interface {
// Name returns the name of the package.
......
......@@ -123,7 +123,7 @@ func TestAdd_Helm(t *testing.T) {
APIVersion: DefaultAPIVersion,
Kind: DefaultKind,
Libraries: LibraryConfigs{
"chart": &LibaryConfig{
"chart": &LibraryConfig{
Version: "2.0.0",
Path: "chart",
},
......
......@@ -152,7 +152,7 @@ func TestFs_FetchRegistrySpec(t *testing.T) {
APIVersion: DefaultAPIVersion,
Kind: "ksonnet.io/registry",
Libraries: LibraryConfigs{
"apache": &LibaryConfig{
"apache": &LibraryConfig{
Path: "apache",
},
},
......
......@@ -155,6 +155,17 @@ func (gh *GitHub) resolveLatestSHA() (string, error) {
return sha, nil
}
// updateLibVersions updates the libraries in a registry spec to present the provided version.
func updateLibVersions(spec *Spec, version string) {
if spec == nil {
return
}
for _, lib := range spec.Libraries {
lib.Version = version
}
}
// FetchRegistrySpec fetches the registry spec (registry.yaml, inventory of packages)
// This inventory may have been previously cached on disk. If the cache is not stale,
// it will be used. Otherwise, the spec is fetched from the remote repository.
......@@ -181,12 +192,14 @@ func (gh *GitHub) FetchRegistrySpec() (*Spec, error) {
if err != nil || sha == "" {
log.Warnf("%v", errors.Wrapf(err, "unable to resolve commit for refspec: %v", gh.hd.refSpec))
log.Warnf("falling back to cached version (%v)", cachedVersion)
updateLibVersions(registrySpec, gh.hd.refSpec)
return registrySpec, nil
}
// Check if cache is still current
if exists && cachedVersion == sha {
log.Debugf("using cache @%v", sha)
updateLibVersions(registrySpec, sha)
return registrySpec, nil
}
......@@ -208,6 +221,7 @@ func (gh *GitHub) FetchRegistrySpec() (*Spec, error) {
if err != nil {
return nil, err
}
updateLibVersions(registrySpec, sha)
var registrySpecBytes []byte
registrySpecBytes, err = registrySpec.Marshal()
......
......@@ -215,9 +215,9 @@ func TestGithub_FetchRegistrySpec_nocache(t *testing.T) {
Kind: "ksonnet.io/registry",
Version: "12345",
Libraries: LibraryConfigs{
"apache": &LibaryConfig{
"apache": &LibraryConfig{
Path: "apache",
Version: "master",
Version: "12345",
},
},
}
......
......@@ -100,7 +100,7 @@ func (h *Helm) FetchRegistrySpec() (*Spec, error) {
return nil, errors.Errorf("entries are invalid")
}
spec.Libraries[name] = &LibaryConfig{
spec.Libraries[name] = &LibraryConfig{
Path: name,
Version: chart.Version,
}
......
......@@ -150,11 +150,11 @@ func TestHelm_FetchRegistrySpec(t *testing.T) {
APIVersion: DefaultAPIVersion,
Kind: DefaultKind,
Libraries: LibraryConfigs{
"app-a": &LibaryConfig{
"app-a": &LibraryConfig{
Path: "app-a",
Version: "0.1.0",
},
"app-b": &LibaryConfig{
"app-b": &LibraryConfig{
Path: "app-b",
Version: "0.2.0",
},
......
......@@ -138,3 +138,26 @@ func (_m *PackageManager) Prototypes() (prototype.Prototypes, error) {
return r0, r1
}
// RemotePackages provides a mock function with given fields:
func (_m *PackageManager) RemotePackages() ([]pkg.Package, error) {
ret := _m.Called()
var r0 []pkg.Package
if rf, ok := ret.Get(0).(func() []pkg.Package); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]pkg.Package)
}
}
var r1 error
if rf, ok := ret.Get(1).(func() error); ok {
r1 = rf()
} else {
r1 = ret.Error(1)
}
return r0, r1
}
......@@ -44,18 +44,26 @@ type PackageManager interface {
// of the specified environment.
PackagesForEnv(e *app.EnvironmentConfig) ([]pkg.Package, error)
// RemotePackages returns a list of remote packages.
RemotePackages() ([]pkg.Package, error)
// Prototypes lists prototypes.
Prototypes() (prototype.Prototypes, error)
InstalledChecker
}
type registryConfigLister interface {
Registries() (app.RegistryConfigs, error)
}
// packageManager is an implementation of PackageManager.
type packageManager struct {
app app.App
InstallChecker pkg.InstallChecker
packagesFn func() ([]pkg.Package, error)
registriesFn func() (map[string]SpecFetcher, error)
}
var _ PackageManager = (*packageManager)(nil)
......@@ -67,10 +75,36 @@ func NewPackageManager(a app.App) PackageManager {
InstallChecker: &pkg.DefaultInstallChecker{App: a},
}
pm.packagesFn = pm.Packages
pm.registriesFn = func() (map[string]SpecFetcher, error) {
return resolveRegistries(a)
}
return &pm
}
// resolveRegistries returns a list of registries from the provided app.
// (SpecFetcher is a subset of the Registry interface)
func resolveRegistries(a app.App) (map[string]SpecFetcher, error) {
if a == nil {
return nil, errors.New("nil app")
}
cfgs, err := a.Registries()
if err != nil {
return nil, err
}
result := make(map[string]SpecFetcher)
for _, cfg := range cfgs {
r, err := Locate(a, cfg)
if err != nil {
return nil, errors.Wrapf(err, "resolving registry: %v", cfg.Name)
}
result[cfg.Name] = r
}
return result, nil
}
// Find finds a package by name. Package names have the format `<registry>/<library>@<version>`.
// Remote registries may be consulted if the package is not installed locally.
func (m *packageManager) Find(name string) (pkg.Package, error) {
......@@ -115,7 +149,7 @@ func (m *packageManager) Find(name string) (pkg.Package, error) {
libraryConfig, ok := libraryConfigs[d.Name]
if ok {
return m.loadPackage(registry.MakeRegistryConfig(), d.Name, d.Registry, libraryConfig.Version)
return m.loadPackage(registry.MakeRegistryConfig(), d.Name, d.Registry, libraryConfig.Version, m.InstallChecker)
}
// TODO - Check libraries configured under environments
......@@ -134,50 +168,47 @@ type remotePackage struct {
partConfig *parts.Spec
}
var _ pkg.Package = (*remotePackage)(nil)
var _ pkg.Package = remotePackage{}
func (p *remotePackage) Name() string {
if p == nil || p.partConfig == nil {
func (p remotePackage) Name() string {
if p.partConfig == nil {
return ""
}
return p.partConfig.Name
}
func (p *remotePackage) RegistryName() string {
if p == nil {
return ""
}
func (p remotePackage) RegistryName() string {
return p.registryName
}
func (p *remotePackage) Version() string {
if p == nil || p.partConfig == nil {
func (p remotePackage) Version() string {
if p.partConfig == nil {
return ""
}
return p.partConfig.Version
}
func (p *remotePackage) Description() string {
if p == nil || p.partConfig == nil {
func (p remotePackage) Description() string {
if p.partConfig == nil {
return ""
}
return p.partConfig.Description
}
func (p *remotePackage) IsInstalled() (bool, error) {
func (p remotePackage) IsInstalled() (bool, error) {
return false, nil
}
func (p *remotePackage) Prototypes() (prototype.Prototypes, error) {
func (p remotePackage) Prototypes() (prototype.Prototypes, error) {
return prototype.Prototypes{}, nil
}
func (p *remotePackage) Path() string {
func (p remotePackage) Path() string {
return ""
}
func (p *remotePackage) String() string {
if p == nil || p.partConfig == nil {
func (p remotePackage) String() string {
if p.partConfig == nil {
return "nil"
}
return fmt.Sprintf("%s/%s@%s", p.registryName, p.partConfig.Name, p.partConfig.Version)
......@@ -210,7 +241,7 @@ func (m *packageManager) Packages() ([]pkg.Package, error) {
libraryConfig.Registry, libraryConfig.Name)
}
p, err := m.loadPackage(registryConfig, libraryConfig.Name, libraryConfig.Registry, libraryConfig.Version)
p, err := m.loadPackage(registryConfig, libraryConfig.Name, libraryConfig.Registry, libraryConfig.Version, pkg.TrueInstallChecker{})
if err != nil {
return nil, err
}
......@@ -262,7 +293,7 @@ func (m *packageManager) PackagesForEnv(e *app.EnvironmentConfig) ([]pkg.Package
libraryConfig.Registry, k)
}
p, err := m.loadPackage(registryConfig, k, libraryConfig.Registry, libraryConfig.Version)
p, err := m.loadPackage(registryConfig, k, libraryConfig.Registry, libraryConfig.Version, pkg.TrueInstallChecker{})
if err != nil {
return nil, err
}
......@@ -273,16 +304,46 @@ func (m *packageManager) PackagesForEnv(e *app.EnvironmentConfig) ([]pkg.Package
return packages, nil
}
func (m *packageManager) loadPackage(registryConfig *app.RegistryConfig, pkgName, registryName, version string) (pkg.Package, error) {
func (m *packageManager) RemotePackages() ([]pkg.Package, error) {
registries, err := m.registriesFn()
if err != nil {
return nil, err
}
var pkgs []pkg.Package
for name, r := range registries {
spec, err := r.FetchRegistrySpec()
if err != nil {
return nil, err
}
for libName, config := range spec.Libraries {
p := remotePackage{
registryName: name,
partConfig: &parts.Spec{
Name: libName,
Version: config.Version,
},
}
pkgs = append(pkgs, p)
}
}
return pkgs, nil
}
func (m *packageManager) loadPackage(registryConfig *app.RegistryConfig, pkgName, registryName, version string, installChecker pkg.InstallChecker) (pkg.Package, error) {