Commit e2b0a883 authored by Oren Shomron's avatar Oren Shomron
Browse files

Garbage collection for vendored packages



Packages are candidates for removal when they are no longer referenced by either a specific environment or the global scope.

Closes #624
Signed-off-by: default avatarOren Shomron <shomron@gmail.com>
parent 200abf0f
......@@ -245,6 +245,7 @@ func withApp(t *testing.T, fn func(*mocks.App)) {
appMock := &mocks.App{}
appMock.On("Fs").Return(fs)
appMock.On("Root").Return("/")
appMock.On("VendorPath").Return("/vendor")
fn(appMock)
}
......
......@@ -21,6 +21,7 @@ import (
"text/template"
"github.com/ksonnet/ksonnet/pkg/app"
"github.com/ksonnet/ksonnet/pkg/pkg"
"github.com/ksonnet/ksonnet/pkg/registry"
)
......@@ -69,7 +70,12 @@ func NewPkgDescribe(m map[string]interface{}) (*PkgDescribe, error) {
// Run describes a package.
func (pd *PkgDescribe) Run() error {
p, err := pd.packageManager.Find(pd.pkgName)
d, err := pkg.Parse(pd.pkgName)
if err != nil {
return err
}
p, err := pd.packageManager.Find(d)
if err != nil {
return err
}
......
......@@ -21,6 +21,7 @@ import (
"testing"
"github.com/ksonnet/ksonnet/pkg/app"
"github.com/ksonnet/ksonnet/pkg/pkg"
"github.com/ksonnet/ksonnet/pkg/prototype"
"github.com/pkg/errors"
......@@ -34,6 +35,7 @@ import (
)
func TestPkgDescribe(t *testing.T) {
d := pkg.Descriptor{Name: "apache"}
cases := []struct {
name string
......@@ -52,7 +54,7 @@ func TestPkgDescribe(t *testing.T) {
p.On("IsInstalled").Return(false, nil)
pkgManager := &regmocks.PackageManager{}
pkgManager.On("Find", "apache").Return(p, nil)
pkgManager.On("Find", d).Return(p, nil)
return pkgManager
},
......@@ -76,7 +78,7 @@ func TestPkgDescribe(t *testing.T) {
p.On("Prototypes").Return(prototypes, nil)
pkgManager := &regmocks.PackageManager{}
pkgManager.On("Find", "apache").Return(p, nil)
pkgManager.On("Find", d).Return(p, nil)
return pkgManager
},
......@@ -86,7 +88,7 @@ func TestPkgDescribe(t *testing.T) {
isErr: true,
pkgManager: func() registry.PackageManager {
pkgManager := &regmocks.PackageManager{}
pkgManager.On("Find", "apache").Return(nil, errors.New("failed"))
pkgManager.On("Find", d).Return(nil, errors.New("failed"))
return pkgManager
},
......@@ -100,7 +102,7 @@ func TestPkgDescribe(t *testing.T) {
p.On("IsInstalled").Return(false, errors.New("failed"))
pkgManager := &regmocks.PackageManager{}
pkgManager.On("Find", "apache").Return(p, nil)
pkgManager.On("Find", d).Return(p, nil)
return pkgManager
},
......@@ -115,7 +117,7 @@ func TestPkgDescribe(t *testing.T) {
p.On("IsInstalled").Return(true, nil)
pkgManager := &regmocks.PackageManager{}
pkgManager.On("Find", "apache").Return(p, nil)
pkgManager.On("Find", d).Return(p, nil)
return pkgManager
},
......@@ -129,7 +131,7 @@ func TestPkgDescribe(t *testing.T) {
p.On("IsInstalled").Return(false, nil)
pkgManager := &regmocks.PackageManager{}
pkgManager.On("Find", "apache").Return(p, nil)
pkgManager.On("Find", d).Return(p, nil)
return pkgManager
},
......
......@@ -46,6 +46,7 @@ type PkgInstall struct {
envName string
force bool
checker registry.InstalledChecker
gc registry.GarbageCollector
libCacherFn libCacher
libUpdateFn libUpdater
envCheckerFn envChecker
......@@ -62,13 +63,16 @@ func NewPkgInstall(m map[string]interface{}) (*PkgInstall, error) {
httpClient := ol.LoadHTTPClient()
httpClientOpt := registry.HTTPClientOpt(httpClient)
pm := registry.NewPackageManager(a, httpClientOpt)
nl := &PkgInstall{
app: a,
libName: ol.LoadString(OptionPkgName),
customName: ol.LoadString(OptionName),
force: ol.LoadBool(OptionForce),
envName: ol.LoadOptionalString(OptionEnvName),
checker: registry.NewPackageManager(a, httpClientOpt),
checker: pm,
gc: registry.NewGarbageCollector(a.Fs(), pm, a.VendorPath()),
libCacherFn: func(a app.App, checker registry.InstalledChecker, d pkg.Descriptor, customName string, force bool) (*app.LibraryConfig, error) {
return registry.CacheDependency(a, checker, d, customName, force, httpClient)
......@@ -123,7 +127,14 @@ func (pi *PkgInstall) Run() error {
return nil
}
// TODO Garbage collector hook here
// Optionally remove any orphaned vendor directories
if err := pi.gc.RemoveOrphans(pkg.Descriptor{
Registry: oldCfg.Registry,
Name: oldCfg.Name,
Version: oldCfg.Version,
}); err != nil {
return errors.Wrapf(err, "garbage collection for package %v", oldCfg)
}
return nil
}
......
......@@ -19,6 +19,8 @@ import (
"github.com/ksonnet/ksonnet/pkg/app"
"github.com/ksonnet/ksonnet/pkg/pkg"
"github.com/ksonnet/ksonnet/pkg/registry"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
// PkgRemove removes packages
......@@ -27,6 +29,7 @@ type PkgRemove struct {
pkgName string
envName string
checker registry.InstalledChecker
gc registry.GarbageCollector
libUpdateFn libUpdater
}
......@@ -39,11 +42,14 @@ func NewPkgRemove(m map[string]interface{}) (*PkgRemove, error) {
return nil, ol.err
}
pm := registry.NewPackageManager(a)
pr := &PkgRemove{
app: a,
pkgName: ol.LoadString(OptionPkgName),
envName: ol.LoadOptionalString(OptionEnvName),
libUpdateFn: a.UpdateLib,
gc: registry.NewGarbageCollector(a.Fs(), pm, a.VendorPath()),
}
if ol.err != nil {
......@@ -79,6 +85,16 @@ func (pr *PkgRemove) Run() error {
return nil
}
// TODO: Garbage collection hook goes here
log.Infof("Removing package %v", oldCfg)
// Optionally remove any orphaned vendor directories
if err := pr.gc.RemoveOrphans(pkg.Descriptor{
Registry: oldCfg.Registry,
Name: oldCfg.Name,
Version: oldCfg.Version,
}); err != nil {
return errors.Wrapf(err, "garbage collection for package %v", oldCfg)
}
return nil
}
......@@ -563,3 +563,19 @@ func (s *Spec) UpdateEnvironmentConfig(name string, env *EnvironmentConfig) erro
s.Environments[env.Name] = env
return nil
}
func (l LibraryConfig) String() string {
switch {
case l.Registry != "" && l.Version != "":
return fmt.Sprintf("%s/%s@%s", l.Registry, l.Name, l.Version)
case l.Registry != "" && l.Version == "":
return fmt.Sprintf("%s/%s", l.Registry, l.Name)
case l.Registry == "" && l.Version != "":
return fmt.Sprintf("%s@%s", l.Name, l.Version)
case l.Registry == "" && l.Version == "":
return l.Name
default:
// Not sure which case we missed, just default to verbose
return fmt.Sprintf("%s/%s@%s", l.Registry, l.Name, l.Version)
}
}
......@@ -198,3 +198,16 @@ func (h *Helm) Path() string {
}
return path
}
// HelmVendorPath returns a path for vendoring the described package.
func HelmVendorPath(a app.App, d Descriptor) string {
if a == nil {
return ""
}
path, err := chartConfigDir(a, d.Name, d.Registry, d.Version)
if err != nil {
return ""
}
return path
}
......@@ -204,3 +204,12 @@ func (l *Local) Path() string {
return buildPath(l.a, l.registryName, l.name, l.version)
}
// LocalVendorPath returns a path for vendoring the described package.
func LocalVendorPath(a app.App, d Descriptor) string {
if a == nil {
return ""
}
return buildPath(a, d.Registry, d.Name, d.Version)
}
......@@ -34,21 +34,42 @@ type Descriptor struct {
Version string
}
func (d Descriptor) String() string {
switch {
case d.Registry != "" && d.Version != "":
return fmt.Sprintf("%s/%s@%s", d.Registry, d.Name, d.Version)
case d.Registry != "" && d.Version == "":
return fmt.Sprintf("%s/%s", d.Registry, d.Name)
case d.Registry == "" && d.Version != "":
return fmt.Sprintf("%s@%s", d.Name, d.Version)
case d.Registry == "" && d.Version == "":
return d.Name
default:
// Not sure which case we missed, just default to verbose
return fmt.Sprintf("%s/%s@%s", d.Registry, d.Name, d.Version)
}
}
// Parse parses a package identifier into its components
// <registry>/<name>@<version>
func Parse(id string) (Descriptor, error) {
var registry, name, version string
matches := reDescriptor.FindStringSubmatch(id)
if len(matches) == 0 {
return Descriptor{}, errInvalidSpec
}
if matches[2] == "" {
return Descriptor{Name: strings.TrimPrefix(matches[1], "/")}, nil
// No registry
name = strings.TrimPrefix(matches[1], "/")
} else {
// Registry and name
registry = matches[1]
name = strings.TrimPrefix(matches[2], "/")
}
registry := matches[1]
name := strings.TrimPrefix(matches[2], "/")
version := strings.TrimPrefix(matches[3], "@")
version = strings.TrimPrefix(matches[3], "@")
return Descriptor{
Registry: registry,
......
......@@ -39,6 +39,10 @@ func Test_Parse(t *testing.T) {
name: "parts-infra/contour@0.1.0",
expected: Descriptor{Registry: "parts-infra", Name: "contour", Version: "0.1.0"},
},
{
name: "contour@0.1.0",
expected: Descriptor{Registry: "", Name: "contour", Version: "0.1.0"},
},
{
name: "@foo/bar@baz@doh",
isErr: true,
......
......@@ -98,12 +98,22 @@ func (ic *DefaultInstallChecker) IsInstalled(name string) (bool, error) {
return false, errors.New("app is nil")
}
d, err := Parse(name)
if err != nil {
return false, errors.Wrapf(err, "parsing package descriptor: %s", name)
}
libs, err := ic.App.Libraries()
if err != nil {
return false, errors.Wrapf(err, "checking if package %q is installed", name)
}
_, isGlobal := libs[name]
var isGlobal bool
if l, ok := libs[d.Name]; ok {
if d.Version == "" || l.Version == d.Version {
isGlobal = true
}
}
envs, err := ic.App.Environments()
if err != nil {
......@@ -112,9 +122,11 @@ func (ic *DefaultInstallChecker) IsInstalled(name string) (bool, error) {
var isLocal bool
for _, e := range envs {
_, isLocal = e.Libraries[name]
if isLocal {
break
if l, ok := e.Libraries[d.Name]; ok {
if d.Version == "" || l.Version == d.Version {
isLocal = true
break
}
}
}
......
......@@ -29,15 +29,17 @@ import (
func Test_DefaultInstallChecker_isInstalled(t *testing.T) {
cases := []struct {
name string
libName string
setupLibraries func(*amocks.App)
isInstalled bool
isErr bool
}{
{
name: "is installed globally",
name: "is installed globally",
libName: "redis",
setupLibraries: func(a *amocks.App) {
libraries := app.LibraryConfigs{
"redis": &app.LibraryConfig{},
"redis": &app.LibraryConfig{Version: "5.0.0"},
}
a.On("Libraries").Return(libraries, nil)
......@@ -46,14 +48,42 @@ func Test_DefaultInstallChecker_isInstalled(t *testing.T) {
isInstalled: true,
},
{
name: "not installed",
name: "is installed globally - match version",
libName: "redis@5.0.0",
setupLibraries: func(a *amocks.App) {
libraries := app.LibraryConfigs{
"redis": &app.LibraryConfig{Version: "5.0.0"},
}
a.On("Libraries").Return(libraries, nil)
a.On("Environments").Return(app.EnvironmentConfigs{}, nil)
},
isInstalled: true,
},
{
name: "is installed globally - version mismatch",
libName: "redis@5.0.1",
setupLibraries: func(a *amocks.App) {
libraries := app.LibraryConfigs{
"redis": &app.LibraryConfig{Version: "5.0.0"},
}
a.On("Libraries").Return(libraries, nil)
a.On("Environments").Return(app.EnvironmentConfigs{}, nil)
},
isInstalled: false,
},
{
name: "not installed",
libName: "redis",
setupLibraries: func(a *amocks.App) {
a.On("Libraries").Return(nil, nil)
a.On("Environments").Return(app.EnvironmentConfigs{}, nil)
},
},
{
name: "libraries error",
name: "libraries error",
libName: "redis",
setupLibraries: func(a *amocks.App) {
a.On("Libraries").Return(nil, errors.New("failed"))
a.On("Environments").Return(app.EnvironmentConfigs{}, nil)
......@@ -61,10 +91,29 @@ func Test_DefaultInstallChecker_isInstalled(t *testing.T) {
isErr: true,
},
{
name: "is installed in an environment",
name: "is installed in an environment",
libName: "redis",
setupLibraries: func(a *amocks.App) {
libraries := app.LibraryConfigs{
"redis": &app.LibraryConfig{},
"redis": &app.LibraryConfig{Version: "5.0.0"},
}
a.On("Libraries").Return(app.LibraryConfigs{}, nil)
a.On("Environments").Return(app.EnvironmentConfigs{
"default": &app.EnvironmentConfig{
Name: "default",
Libraries: libraries,
},
}, nil)
},
isInstalled: true,
},
{
name: "is installed in an environment - match version",
libName: "redis@5.0.0",
setupLibraries: func(a *amocks.App) {
libraries := app.LibraryConfigs{
"redis": &app.LibraryConfig{Version: "5.0.0"},
}
a.On("Libraries").Return(app.LibraryConfigs{}, nil)
......@@ -78,7 +127,26 @@ func Test_DefaultInstallChecker_isInstalled(t *testing.T) {
isInstalled: true,
},
{
name: "is installed both globally and in environment",
name: "is installed in an environment - version mismatch",
libName: "redis@5.0.1",
setupLibraries: func(a *amocks.App) {
libraries := app.LibraryConfigs{
"redis": &app.LibraryConfig{Version: "5.0.0"},
}
a.On("Libraries").Return(app.LibraryConfigs{}, nil)
a.On("Environments").Return(app.EnvironmentConfigs{
"default": &app.EnvironmentConfig{
Name: "default",
Libraries: libraries,
},
}, nil)
},
isInstalled: false,
},
{
name: "is installed both globally and in environment",
libName: "redis",
setupLibraries: func(a *amocks.App) {
libraries := app.LibraryConfigs{
"redis": &app.LibraryConfig{},
......@@ -103,7 +171,7 @@ func Test_DefaultInstallChecker_isInstalled(t *testing.T) {
ic := DefaultInstallChecker{App: a}
i, err := ic.IsInstalled("redis")
i, err := ic.IsInstalled(tc.libName)
if tc.isErr {
require.Error(t, err)
return
......
......@@ -152,16 +152,11 @@ func versionAndVendorRelPath(lib *app.LibraryConfig, vendorRoot string, relPath
// Version the path
var versionedPath string
if lib.Version != "" {
//filepath.ToSlash()
parts := strings.SplitN(filepath.ToSlash(relPath), "/", -1)
if parts[0] == lib.Name {
parts[0] = fmt.Sprintf("%s@%s", lib.Name, lib.Version)
}
versionedPath = filepath.FromSlash(strings.Join(parts, "/"))
// oldPrefix := filepath.Join(lib.Registry, lib.Name)
// newPrefix := fmt.Sprintf("%s@%s", lib.Name, lib.Version)
// versionedPath = strings.Replace(relPath, oldPrefix, newPrefix, 1)
} else {
// For unversioned packages, use path as-is
versionedPath = relPath
......
// Copyright 2018 The ksonnet authors
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package registry
import (
"io"
"path/filepath"
"strings"
"github.com/ksonnet/ksonnet/pkg/pkg"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/afero"
)
// FsRemoveAller Subset of afero.Fs - just remove a directory
type FsRemoveAller interface {
// RemoveAll removes a directory path and any children it contains. It
// does not fail if the path does not exist (return nil).
RemoveAll(path string) error
}
type vendorPathResolver interface {
InstalledChecker
VendorPath(pkg.Descriptor) (string, error)
}
// GarbageCollector removes vendored packages that are no longer needed
type GarbageCollector struct {
pkgManager vendorPathResolver
fs FsRemoveAller
root string
removeEmptyParentsFn func(path string, root string) error
}
// NewGarbageCollector constructs a GarbageCollector
func NewGarbageCollector(fs afero.Fs, pm vendorPathResolver, root string) GarbageCollector {
return GarbageCollector{
pkgManager: pm,
fs: fs,
root: root,
removeEmptyParentsFn: func(path string, root string) error {
return removeEmptyParents(fs, path, root)
},
}
}
// RemoveOrphans removes vendored packages that have been orphaned
func (gc GarbageCollector) RemoveOrphans(d pkg.Descriptor) error {
log := log.WithField("action", "GarbageCollector.RemoveOrphans")
installed, err := gc.pkgManager.IsInstalled(d)
if err != nil {
return errors.Wrapf(err, "checking installed status: %v", d)
}
// Only remove orphans
if installed {
return nil
}
path, err := gc.pkgManager.VendorPath(d)
if err != nil {
return errors.Wrapf(err, "resolving path for descriptor: %v", d)
}
if path == "" {
return nil
}
if gc.fs == nil {
return errors.New("nil fs")
}
log.Debugf("removing path %s", path)
if err := gc.fs.RemoveAll(path); err != nil {
return errors.Wrapf(err, "removing path %s for package %v", path, d)
}
if gc.removeEmptyParentsFn == nil {
return nil
}
if err := gc.removeEmptyParentsFn(path, gc.root); err != nil {
return errors.Wrapf(err, "removing empty parents path %s", path)
}
return nil
}
// Returns true if the specified directory is empty
func isDirEmpty(fs afero.Fs, path string) (bool, error) {
if fs == nil {
return false, errors.New("nil fs")
}
f, err := fs.Open(path)
if err != nil {
return false, err
}
defer f.Close()
_, err = f.Readdirnames(1)
if err == io.EOF {
return true, nil
}
return false, err
}
// Remove empty directories, up to the provided root, exclusive
func removeEmptyParents(fs afero.Fs, path string, root string) error {
if fs == nil {
return errors.New("nil fs")
}
root = filepath.Clean(root)