Unverified Commit c2bef13d authored by bryanl's avatar bryanl
Browse files

Move/upgrade ks lib location

Moving generated ksonnet lib from lib/<verson> to lib/ksonnet-lib/<version>.
This change will free up lib to be used for other lib type things.

Ksonnet will warn if ksonnet-lib is in the legacy location and the
user can use `ks upgrade` to move the files to their new location.

Also:

* update Makefile to search harder for apimachiner revision

Signed-off-by: bryanl bryanliles@gmail.com
parent 08ec7e99
......@@ -85,5 +85,5 @@ func genLib(a app.App, k8sSpecFlag, libPath string) error {
return err
}
return libManager.GenerateLibData(false)
return libManager.GenerateLibData()
}
......@@ -124,13 +124,13 @@ func Load(fs afero.Fs, cwd string, skipFindRoot bool) (App, error) {
}
}
func updateLibData(fs afero.Fs, k8sSpecFlag, libPath string, useVersionPath bool) (string, error) {
func updateLibData(fs afero.Fs, k8sSpecFlag, libPath string) (string, error) {
lm, err := lib.NewManager(k8sSpecFlag, fs, libPath)
if err != nil {
return "", err
}
if err := lm.GenerateLibData(useVersionPath); err != nil {
if err := lm.GenerateLibData(); err != nil {
return "", err
}
......@@ -142,7 +142,7 @@ func app010LibPath(root string) string {
}
// StubUpdateLibData always returns no error.
func StubUpdateLibData(fs afero.Fs, k8sSpecFlag, libPath string, useVersionPath bool) (string, error) {
func StubUpdateLibData(fs afero.Fs, k8sSpecFlag, libPath string) (string, error) {
return "v1.8.7", nil
}
......
......@@ -73,7 +73,7 @@ func (a *App001) AddEnvironment(name, k8sSpecFlag string, spec *EnvironmentSpec,
return err
}
_, err = LibUpdater(a.fs, k8sSpecFlag, a.appLibPath(name), false)
_, err = LibUpdater(a.fs, k8sSpecFlag, a.appLibPath(name))
return err
}
......@@ -287,7 +287,7 @@ func (a *App001) convertEnvironment(envName string, dryRun bool) error {
}
k8sSpecFlag := fmt.Sprintf("version:%s", env.KubernetesVersion)
_, err = LibUpdater(a.fs, k8sSpecFlag, app010LibPath(a.root), true)
_, err = LibUpdater(a.fs, k8sSpecFlag, app010LibPath(a.root))
if err != nil {
return err
}
......
......@@ -253,7 +253,7 @@ func TestApp001_UpdateTargets(t *testing.T) {
func withApp001Fs(t *testing.T, appName string, fn func(app *App001)) {
ogLibUpdater := LibUpdater
LibUpdater = func(fs afero.Fs, k8sSpecFlag string, libPath string, useVersionPath bool) (string, error) {
LibUpdater = func(fs afero.Fs, k8sSpecFlag string, libPath string) (string, error) {
path := filepath.Join(libPath, "swagger.json")
stageFile(t, fs, "swagger.json", path)
return "v1.8.7", nil
......
......@@ -17,8 +17,10 @@ package app
import (
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"strings"
"github.com/ksonnet/ksonnet/pkg/lib"
......@@ -31,6 +33,9 @@ import (
// App010 is a ksonnet 0.1.0 application.
type App010 struct {
*baseApp
out io.Writer
libPaths map[string]string
}
var _ App = (*App010)(nil)
......@@ -41,6 +46,9 @@ func NewApp010(fs afero.Fs, root string) *App010 {
a := &App010{
baseApp: ba,
out: os.Stdout,
libPaths: make(map[string]string),
}
return a
......@@ -59,7 +67,7 @@ func (a *App010) AddEnvironment(name, k8sSpecFlag string, newEnv *EnvironmentSpe
}
if k8sSpecFlag != "" {
ver, err := LibUpdater(a.fs, k8sSpecFlag, app010LibPath(a.root), true)
ver, err := LibUpdater(a.fs, k8sSpecFlag, app010LibPath(a.root))
if err != nil {
return err
}
......@@ -139,6 +147,10 @@ func (a *App010) Init() error {
// LibPath returns the lib path for an env environment.
func (a *App010) LibPath(envName string) (string, error) {
if lp, ok := a.libPaths[envName]; ok {
return lp, nil
}
env, err := a.Environment(envName)
if err != nil {
return "", err
......@@ -150,7 +162,24 @@ func (a *App010) LibPath(envName string) (string, error) {
return "", err
}
return lm.GetLibPath(true)
lp, err := lm.GetLibPath()
if err != nil {
return "", err
}
a.checkKsonnetLib(lp)
a.libPaths[envName] = lp
return lp, nil
}
func (a *App010) checkKsonnetLib(lp string) {
libRoot := filepath.Join(a.Root(), LibDirName, "ksonnet-lib")
if !strings.HasPrefix(lp, libRoot) {
logrus.Warnf("ksonnet has moved ksonnet-lib paths to %q. The current location of "+
"of your existing ksonnet-libs can be automatically moved by ksonnet with `ks upgrade`",
libRoot)
}
}
// Libraries returns application libraries.
......@@ -239,6 +268,55 @@ func (a *App010) UpdateTargets(envName string, targets []string) error {
// Upgrade upgrades the app to the latest apiVersion.
func (a *App010) Upgrade(dryRun bool) error {
if err := a.checkForOldKSLibLocation(dryRun); err != nil {
return err
}
return nil
}
var (
// reKSLibName matches a ksonnet library directory e.g. v1.10.3.
reKSLibName = regexp.MustCompile(`^v\d+\.\d+\.\d+$`)
)
func (a *App010) checkForOldKSLibLocation(dryRun bool) error {
libRoot := filepath.Join(a.Root(), LibDirName)
fis, err := afero.ReadDir(a.Fs(), libRoot)
if err != nil {
return err
}
if dryRun {
fmt.Fprintf(a.out, "[dry run] Updating ksonnet-lib paths\n")
}
if err = a.fs.MkdirAll(filepath.Join(libRoot, lib.KsonnetLibHome), DefaultFolderPermissions); err != nil {
return err
}
for _, fi := range fis {
if !fi.IsDir() {
continue
}
if reKSLibName.MatchString(fi.Name()) {
p := filepath.Join(libRoot, fi.Name())
new := filepath.Join(libRoot, lib.KsonnetLibHome, fi.Name())
if dryRun {
fmt.Fprintf(a.out, "[dry run] Moving %q from %s to %s\n", fi.Name(), p, new)
continue
}
fmt.Fprintf(a.out, "Moving %q from %s to %s\n", fi.Name(), p, new)
err = a.fs.Rename(p, new)
if err != nil {
return errors.Wrapf(err, "renaming %s to %s", p, new)
}
}
}
return nil
}
......
......@@ -177,7 +177,7 @@ func TestApp0101_LibPath(t *testing.T) {
path, err := app.LibPath("default")
require.NoError(t, err)
expected := filepath.Join("/", "lib", "v1.7.0")
expected := filepath.Join("/", "lib", "ksonnet-lib", "v1.7.0")
require.Equal(t, expected, path)
})
}
......@@ -263,10 +263,81 @@ func TestApp010_RenameEnvironment(t *testing.T) {
}
func TestApp010_Upgrade(t *testing.T) {
withApp010Fs(t, "app010_app.yaml", func(app *App010) {
err := app.Upgrade(false)
require.NoError(t, err)
})
cases := []struct {
name string
init func(t *testing.T, app *App010)
checkUpgrade func(t *testing.T, app *App010)
dryRun bool
isErr bool
}{
{
name: "ksonnet lib doesn't need to be upgraded",
init: func(t *testing.T, app *App010) {
err := app.Fs().MkdirAll("/lib", DefaultFolderPermissions)
require.NoError(t, err)
p := filepath.Join(app.Root(), "lib", "ksonnet-lib", "v1.10.3")
err = app.Fs().MkdirAll(p, DefaultFolderPermissions)
require.NoError(t, err)
},
dryRun: false,
},
{
name: "ksonnet lib needs to be upgraded",
init: func(t *testing.T, app *App010) {
err := app.Fs().MkdirAll("/lib", DefaultFolderPermissions)
require.NoError(t, err)
p := filepath.Join(app.Root(), "lib", "v1.10.3")
err = app.Fs().MkdirAll(p, DefaultFolderPermissions)
require.NoError(t, err)
},
dryRun: false,
},
{
name: "ksonnet lib needs to be upgraded - dry run",
init: func(t *testing.T, app *App010) {
err := app.Fs().MkdirAll("/lib", DefaultFolderPermissions)
require.NoError(t, err)
p := filepath.Join(app.Root(), "lib", "v1.10.3")
err = app.Fs().MkdirAll(p, DefaultFolderPermissions)
require.NoError(t, err)
},
checkUpgrade: func(t *testing.T, app *App010) {
isDir, err := afero.IsDir(app.Fs(), filepath.Join("/lib", "v1.10.3"))
require.NoError(t, err)
require.True(t, isDir)
},
dryRun: true,
},
{
name: "lib doesn't exist",
isErr: true,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
withApp010Fs(t, "app010_app.yaml", func(app *App010) {
if tc.init != nil {
tc.init(t, app)
}
err := app.Upgrade(tc.dryRun)
if tc.isErr {
require.Error(t, err)
return
}
require.NoError(t, err)
if tc.checkUpgrade != nil {
tc.checkUpgrade(t, app)
}
})
})
}
}
func TestApp0101_UpdateTargets(t *testing.T) {
......@@ -284,7 +355,7 @@ func TestApp0101_UpdateTargets(t *testing.T) {
func withApp010Fs(t *testing.T, appName string, fn func(app *App010)) {
ogLibUpdater := LibUpdater
LibUpdater = func(fs afero.Fs, k8sSpecFlag string, libPath string, useVersionPath bool) (string, error) {
LibUpdater = func(fs afero.Fs, k8sSpecFlag string, libPath string) (string, error) {
return "v1.8.7", nil
}
......
......@@ -137,7 +137,7 @@ func assertExists(t *testing.T, fs afero.Fs, path string) {
func assertLib(t *testing.T, fs afero.Fs, root, version string) {
files := []string{"swagger.json", "k.libsonnet", "k8s.libsonnet"}
for _, f := range files {
path := filepath.Join(root, "lib", version, f)
path := filepath.Join(root, "lib", "ksonnet-lib", version, f)
assertExists(t, fs, path)
}
}
......@@ -30,7 +30,7 @@ type parseSuccess struct {
func TestClusterSpecParsingSuccess(t *testing.T) {
testFS := afero.NewMemMapFs()
afero.WriteFile(testFS, blankSwagger, []byte(blankSwaggerData), os.ModePerm)
afero.WriteFile(testFS, swaggerLocation, []byte(blankSwaggerData), os.ModePerm)
var successTests = []parseSuccess{
{"version:v1.7.1", &clusterSpecVersion{"v1.7.1"}},
......@@ -89,7 +89,7 @@ func TestClusterSpecParsingFailure(t *testing.T) {
for _, test := range failureTests {
testFS := afero.NewMemMapFs()
afero.WriteFile(testFS, blankSwagger, []byte(blankSwaggerData), os.ModePerm)
afero.WriteFile(testFS, swaggerLocation, []byte(blankSwaggerData), os.ModePerm)
_, err := ParseClusterSpec(test.input, testFS)
if err == nil {
t.Errorf("Cluster spec parse for '%s' should have failed, but succeeded", test.input)
......
......@@ -16,11 +16,11 @@
package lib
import (
"fmt"
"os"
"path"
"path/filepath"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/afero"
......@@ -34,8 +34,33 @@ const (
// ExtensionsLibFilename is the file name with the contents of the
// generated ksonnet-lib
ExtensionsLibFilename = "k.libsonnet"
// KsonnetLibHome is name of the directory ksonnet lib files will be generated in.
KsonnetLibHome = "ksonnet-lib"
)
// KsLibGenerator generates ksonnet-lib.
type KsLibGenerator interface {
Generate() (*kslib.KsonnetLib, error)
}
type defaultKsLibGenerator struct {
spec ClusterSpec
}
func (g *defaultKsLibGenerator) Generate() (*kslib.KsonnetLib, error) {
if g.spec == nil {
return nil, errors.Errorf("uninitialized ClusterSpec")
}
b, err := g.spec.OpenAPI()
if err != nil {
return nil, err
}
return kslib.Ksonnet(b)
}
// Manager operates on the files in the lib directory of a ksonnet project.
// This included generating the ksonnet-lib files needed to compile the
// ksonnet (jsonnet) code.
......@@ -43,9 +68,10 @@ type Manager struct {
// K8sVersion is the Kubernetes version of the Open API spec.
K8sVersion string
spec ClusterSpec
libPath string
fs afero.Fs
generator KsLibGenerator
}
// NewManager creates a new instance of lib.Manager
......@@ -63,33 +89,25 @@ func NewManager(k8sSpecFlag string, fs afero.Fs, libPath string) (*Manager, erro
return nil, err
}
return &Manager{K8sVersion: version, fs: fs, libPath: libPath, spec: spec}, nil
return &Manager{
K8sVersion: version,
fs: fs,
libPath: libPath,
generator: &defaultKsLibGenerator{spec: spec},
}, nil
}
// GenerateLibData will generate the swagger and ksonnet-lib files in the lib
// directory of a ksonnet project. The swagger and ksonnet-lib files are
// unique to each Kubernetes API version. If the files already exist for a
// specific Kubernetes API version, they won't be re-generated here.
func (m *Manager) GenerateLibData(useVersionPath bool) error {
if m.spec == nil {
return fmt.Errorf("Uninitialized ClusterSpec")
}
b, err := m.spec.OpenAPI()
func (m *Manager) GenerateLibData() error {
kl, err := m.generator.Generate()
if err != nil {
return err
}
kl, err := kslib.Ksonnet(b)
if err != nil {
return err
}
genPath := m.libPath
if useVersionPath {
genPath = filepath.Join(m.libPath, m.K8sVersion)
}
genPath := filepath.Join(m.ksLibDir(), m.K8sVersion)
ok, err := afero.DirExists(m.fs, genPath)
if err != nil {
......@@ -141,21 +159,33 @@ func (m *Manager) GenerateLibData(useVersionPath bool) error {
// GetLibPath returns the absolute path pointing to the directory with the
// metadata files for the provided k8sVersion.
func (m *Manager) GetLibPath(useVersionPath bool) (string, error) {
path := filepath.Join(m.libPath, m.K8sVersion)
ok, err := afero.DirExists(m.fs, string(path))
func (m *Manager) GetLibPath() (string, error) {
basePath := m.ksLibDir()
ok, err := afero.DirExists(m.fs, basePath)
if err != nil {
return "", err
}
if !ok {
log.Debugf("Expected lib directory '%s' but was not found", m.K8sVersion)
// create the directory
if err = m.GenerateLibData(useVersionPath); err != nil {
if err = m.GenerateLibData(); err != nil {
return "", err
}
return path, nil
}
return path, err
return filepath.Join(basePath, m.K8sVersion), nil
}
func (m *Manager) ksLibDir() string {
ksLibPath := filepath.Join(m.libPath, m.K8sVersion)
exists, _ := afero.IsDir(m.fs, ksLibPath)
if exists {
return m.libPath
}
return filepath.Join(m.libPath, "ksonnet-lib")
}
......@@ -20,12 +20,15 @@ import (
"path/filepath"
"testing"
"github.com/ksonnet/ksonnet/pkg/util/kslib"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const (
blankSwagger = "/blankSwagger.json"
swaggerLocation = "/blankSwagger.json"
blankSwaggerData = `{
"swagger": "2.0",
"info": {
......@@ -41,54 +44,120 @@ const (
func TestGenerateLibData(t *testing.T) {
cases := []struct {
name string
useVersionPath bool
basePath string
name string
basePath string
generator KsLibGenerator
swaggerData []byte
}{
{
name: "use version path",
useVersionPath: true,
basePath: "v1.7.0",
},
{
name: "don't use version path",
name: "use version path",
basePath: "v1.7.0",
generator: &fakeKsLibGenerator{
ksonnetLib: &kslib.KsonnetLib{},
},
swaggerData: []byte(blankSwaggerData),
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
testFS := afero.NewMemMapFs()
afero.WriteFile(testFS, blankSwagger, []byte(blankSwaggerData), os.ModePerm)
fs := afero.NewMemMapFs()
afero.WriteFile(fs, swaggerLocation, tc.swaggerData, os.ModePerm)
specFlag := fmt.Sprintf("file:%s", blankSwagger)
specFlag := fmt.Sprintf("file:%s", swaggerLocation)
libPath := "lib"
libManager, err := NewManager(specFlag, testFS, libPath)
if err != nil {
t.Fatal("Failed to initialize lib.Manager")
}
libManager, err := NewManager(specFlag, fs, libPath)
require.NoError(t, err)
err = libManager.GenerateLibData(tc.useVersionPath)
if err != nil {
t.Fatal("Failed to generate lib data")
}
libManager.generator = tc.generator
err = libManager.GenerateLibData()
require.NoError(t, err)
// Verify contents of lib.
genPath := filepath.Join(libPath, tc.basePath)
schemaPath := filepath.Join(genPath, "swagger.json")
extPath := filepath.Join(genPath, "k.libsonnet")
k8sPath := filepath.Join(genPath, "k8s.libsonnet")
checkExists(t, testFS, schemaPath)
checkExists(t, testFS, extPath)
checkExists(t, testFS, k8sPath)
genPath := filepath.Join(libPath, KsonnetLibHome, tc.basePath)
checkKsLib(t, fs, genPath)
})
}
}
func checkKsLib(t *testing.T, fs afero.Fs, path string) {
files := []string{"swagger.json", "k.libsonnet", "k8s.libsonnet"}
for _, f := range files {
p := filepath.Join(path, f)
exists, err := afero.Exists(fs, p)
assert.NoError(t, err, p)
assert.True(t, exists, "%q did not exist", p)
}
}
func checkExists(t *testing.T, fs afero.Fs, path string) {
exists, err := afero.Exists(fs, path)
require.NoError(t, err)
require.True(t, exists, "%q did not exist", path)
func TestManager_GetLibPath(t *testing.T) {
cases := []struct {
name string
initFs func(*testing.T, string, string) afero.Fs
expected string
}{
{
name: "with ksonnet-lib",
initFs: func(t *testing.T, version, libPath string) afero.Fs {
fs := afero.NewMemMapFs()
klPath := filepath.Join(libPath, KsonnetLibHome, version)
err := fs.MkdirAll(klPath, 0755)
require.NoError(t, err)
return fs
},
expected: filepath.FromSlash("lib/ksonnet-lib/v1.10.3"),
},
{
name: "without ksonnet-lib",
initFs: func(t *testing.T, version, libPath string) afero.Fs {
fs := afero.NewMemMapFs()
klPath := filepath.Join(libPath, version)
err := fs.MkdirAll(klPath, 0755)
require.NoError(t, err)
return fs
},
expected: filepath.FromSlash("lib/v1.10.3"),
},
{
name: "doesn't already exist",
initFs: func(t *testing.T, version, libPath string) afero.Fs {
fs := afero.NewMemMapFs()
return fs
},
expected: filepath.FromSlash("lib/ksonnet-lib/v1.10.3"),
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
specFlag := "version:v1.10.3"
libPath := "lib"
fs := tc.initFs(t, "v1.10.3", "lib")
libManager, err := NewManager(specFlag, fs, libPath)
require.NoError(t, err)