Allow for regeneration of lib and registry cache

`ks` will regenerate `lib` and `.ksonnet/registries` if needed. This
means they can be ignored by git.

Fixes #55

Signed-off-by: bryanl <bryanliles@gmail.com>

actions/init.go

@@ -37,8 +37,11 @@ func RunInit(m map[string]interface{}) error {
 	return i.Run()
 }
 
+type appLoadFn func(fs afero.Fs, root string, skipFindRoot bool) (app.App, error)
+
 type appInitFn func(fs afero.Fs, name, rootPath, k8sSpecFlag, serverURI, namespace string, registries []registry.Registry) error
-type initIncubatorFn func() (registry.Registry, error)
+
+type initIncubatorFn func(app.App) (registry.Registry, error)
 
 // Init creates a component namespace
 type Init struct {
@@ -51,6 +54,7 @@ type Init struct {
 	skipDefaultRegistries bool
 
 	appInitFn       appInitFn
+	appLoadFn       appLoadFn
 	initIncubatorFn initIncubatorFn
 }
 
@@ -68,6 +72,7 @@ func NewInit(m map[string]interface{}) (*Init, error) {
 		skipDefaultRegistries: ol.loadBool(OptionSkipDefaultRegistries),
 
 		appInitFn:       appinit.Init,
+		appLoadFn:       app.Load,
 		initIncubatorFn: initIncubator,
 	}
 
@@ -83,7 +88,12 @@ func (i *Init) Run() error {
 	var registries []registry.Registry
 
 	if !i.skipDefaultRegistries {
-		gh, err := i.initIncubatorFn()
+		a, err := i.appLoadFn(i.fs, i.rootPath, true)
+		if err != nil {
+			return err
+		}
+
+		gh, err := i.initIncubatorFn(a)
 		if err != nil {
 			return err
 		}
@@ -102,10 +112,12 @@ func (i *Init) Run() error {
 	)
 }
 
-func initIncubator() (registry.Registry, error) {
-	return registry.NewGitHub(&app.RegistryRefSpec{
-		Name:     "incubator",
-		Protocol: registry.ProtocolGitHub,
-		URI:      defaultIncubatorURI,
-	})
+func initIncubator(a app.App) (registry.Registry, error) {
+	return registry.NewGitHub(
+		a,
+		&app.RegistryRefSpec{
+			Name:     "incubator",
+			Protocol: registry.ProtocolGitHub,
+			URI:      defaultIncubatorURI,
+		})
 }

actions/init_test.go

@@ -18,6 +18,7 @@ package actions
 import (
 	"testing"
 
+	"github.com/ksonnet/ksonnet/metadata/app"
 	amocks "github.com/ksonnet/ksonnet/metadata/app/mocks"
 	"github.com/ksonnet/ksonnet/pkg/registry"
 	rmocks "github.com/ksonnet/ksonnet/pkg/registry/mocks"
@@ -85,7 +86,11 @@ func TestInit(t *testing.T) {
 					return nil
 				}
 
-				a.initIncubatorFn = func() (registry.Registry, error) {
+				a.appLoadFn = func(fs afero.Fs, root string, skipFindRoot bool) (app.App, error) {
+					return appMock, nil
+				}
+
+				a.initIncubatorFn = func(a app.App) (registry.Registry, error) {
 					r := &rmocks.Registry{}
 					r.On("Protocol").Return("github")
 					r.On("URI").Return("github.com/ksonnet/parts/tree/master/incubator")

cmd/root.go

@@ -102,7 +102,7 @@ application configuration to remote clusters.
 			isInit = true
 		}
 
-		ka, err = app.Load(appFs, wd)
+		ka, err = app.Load(appFs, wd, false)
 		if err != nil && isInit {
 			return err
 		}

e2e/init_test.go

+// Copyright 2018 The ksonnet authors
+//
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+
 // +build e2e
 
 package e2e
@@ -22,6 +37,10 @@ var _ = Describe("ks init", func() {
 			o := a.registryList()
 			assertOutput(filepath.Join("init", "registry-output.txt"), o.stdout)
 		})
+
+		It("creates a gitignore", func() {
+			assertContents(filepath.Join("init", "gitignore"), filepath.Join(a.dir, ".gitignore"))
+		})
 	})
 
 	Context("without default registries", func() {

e2e/pkg_test.go

@@ -18,6 +18,7 @@
 package e2e
 
 import (
+	"os"
 	"path/filepath"
 
 	. "github.com/onsi/ginkgo"
@@ -101,6 +102,19 @@ var _ = Describe("ks pkg", func() {
 			assertExitStatus(o, 0)
 			assertOutput("pkg/list/output.txt", o.stdout)
 		})
+
+		Context("git spec cache has been been deleted", func() {
+			JustBeforeEach(func() {
+				err := os.RemoveAll(filepath.Join(a.dir, ".ksonnet"))
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			It("generates spec cache", func() {
+				o := a.runKs("pkg", "list")
+				assertExitStatus(o, 0)
+				assertOutput("pkg/list/output.txt", o.stdout)
+			})
+		})
 	})
 
 	Context("use", func() {

e2e/show_test.go

@@ -18,7 +18,11 @@
 package e2e
 
 import (
+	"os"
+	"path/filepath"
+
 	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 )
 
 var _ = Describe("ks show", func() {
@@ -35,4 +39,17 @@ var _ = Describe("ks show", func() {
 		assertOutput("show/output.txt", o.stdout)
 	})
 
+	Context("lib does not exists", func() {
+		JustBeforeEach(func() {
+			err := os.RemoveAll(filepath.Join(a.dir, "lib"))
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("generates spec cache", func() {
+			o := a.runKs("show", "default")
+			assertExitStatus(o, 0)
+			assertOutput("show/output.txt", o.stdout)
+		})
+	})
+
 })

e2e/testdata/output/init/gitignore

+/lib
+/.ksonnet/registries

metadata/app/app.go

@@ -71,15 +71,19 @@ type App interface {
 }
 
 // Load loads the application configuration.
-func Load(fs afero.Fs, cwd string) (App, error) {
-	appRoot, err := findRoot(fs, cwd)
-	if err != nil {
-		return nil, err
+func Load(fs afero.Fs, cwd string, skipFindRoot bool) (App, error) {
+	appRoot := cwd
+	if !skipFindRoot {
+		var err error
+		appRoot, err = findRoot(fs, cwd)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	spec, err := read(fs, appRoot)
 	if err != nil {
-		return nil, err
+		return NewApp010(fs, appRoot), nil
 	}
 
 	switch spec.APIVersion {

metadata/manager.go

@@ -146,7 +146,7 @@ func (m *manager) Root() string {
 }
 
 func (m *manager) App() (app.App, error) {
-	return app.Load(m.appFS, m.rootPath)
+	return app.Load(m.appFS, m.rootPath, false)
 }
 
 func (m *manager) LibPaths() (envPath, vendorPath string) {

pkg/appinit/init.go

@@ -65,7 +65,7 @@ func (i *initApp) Run() error {
 	}
 
 	// Load application.
-	a, err := app.Load(i.fs, i.rootPath)
+	a, err := app.Load(i.fs, i.rootPath, false)
 	if err != nil {
 		return err
 	}
@@ -187,6 +187,10 @@ func (i *initApp) createAppDirTree() error {
 		path    string
 		content []byte
 	}{
+		{
+			filepath.Join(i.rootPath, ".gitignore"),
+			ignoreData,
+		},
 		{
 			filepath.Join(i.rootPath, "components", "params.libsonnet"),
 			component.GenParamsContent(),
@@ -210,3 +214,7 @@ func (i *initApp) createAppDirTree() error {
 
 	return nil
 }
+
+var ignoreData = []byte(`/lib
+/.ksonnet/registries
+`)

pkg/appinit/init_test.go

@@ -94,6 +94,7 @@ func TestInit(t *testing.T) {
 
 func checkApp(t *testing.T, fs afero.Fs, rootPath, version, namespace string) {
 	expectedDirs := []string{
+		".gitignore",
 		"app.yaml",
 		filepath.Join(".ksonnet", "registries", "testdata", "registry.yaml"),
 		filepath.Join("components", "params.libsonnet"),

pkg/registry/add.go

@@ -16,8 +16,6 @@
 package registry
 
 import (
-	"path/filepath"
-
 	"github.com/ksonnet/ksonnet/metadata/app"
 	"github.com/pkg/errors"
 	"github.com/spf13/afero"
@@ -37,7 +35,7 @@ func Add(a app.App, name, protocol, uri, version string, isOverride bool) (*Spec
 
 	switch protocol {
 	case ProtocolGitHub:
-		r, err = githubFactory(initSpec)
+		r, err = githubFactory(a, initSpec)
 	case ProtocolFilesystem:
 		r, err = NewFs(a, initSpec)
 	default:
@@ -54,7 +52,7 @@ func Add(a app.App, name, protocol, uri, version string, isOverride bool) (*Spec
 	}
 
 	// Retrieve the contents of registry.
-	registrySpec, err := getOrCacheRegistry(a, r)
+	registrySpec, err := r.FetchRegistrySpec()
 	if err != nil {
 		return nil, errors.Wrap(err, "cache registry")
 	}
@@ -62,46 +60,6 @@ func Add(a app.App, name, protocol, uri, version string, isOverride bool) (*Spec
 	return registrySpec, nil
 }
 
-func getOrCacheRegistry(a app.App, gh Registry) (*Spec, error) {
-	// Check local disk cache.
-	registrySpecFile := makePath(a, gh)
-	registrySpec, exists, err := load(a, registrySpecFile)
-	if err != nil {
-		return nil, errors.Wrap(err, "load registry spec file")
-	}
-
-	if !exists {
-		// If failed, use the protocol to try to retrieve app specification.
-		registrySpec, err = gh.FetchRegistrySpec()
-		if err != nil {
-			return nil, err
-		}
-
-		registrySpecBytes, err := registrySpec.Marshal()
-		if err != nil {
-			return nil, err
-		}
-
-		// NOTE: We call mkdir after getting the registry spec, since a
-		// network call might fail and leave this half-initialized empty
-		// directory.
-		registrySpecDir := filepath.Join(root(a), gh.RegistrySpecDir())
-		err = a.Fs().MkdirAll(registrySpecDir, app.DefaultFolderPermissions)
-		if err != nil {
-			return nil, err
-		}
-
-		err = afero.WriteFile(a.Fs(), registrySpecFile, registrySpecBytes, app.DefaultFilePermissions)
-		if err != nil {
-			return nil, err
-		}
-	} else if err != nil {
-		return nil, err
-	}
-
-	return registrySpec, nil
-}
-
 func load(a app.App, path string) (*Spec, bool, error) {
 	exists, err := afero.Exists(a.Fs(), path)
 	if err != nil {

pkg/registry/add_test.go

@@ -69,8 +69,8 @@ func TestAdd(t *testing.T) {
 			Return(registryContent, nil, nil)
 
 		ghOpt := GitHubClient(ghMock)
-		githubFactory = func(registryRef *app.RegistryRefSpec) (*GitHub, error) {
-			return NewGitHub(registryRef, ghOpt)
+		githubFactory = func(a app.App, registryRef *app.RegistryRefSpec) (*GitHub, error) {
+			return NewGitHub(a, registryRef, ghOpt)
 		}
 
 		spec, err := Add(appMock, "new", ProtocolGitHub, "github.com/foo/bar", "", true)

pkg/registry/cache.go

@@ -26,7 +26,7 @@ import (
 	"github.com/spf13/afero"
 )
 
-// CacheDependency caches registry dependencies.
+// CacheDependency vendors registry dependencies.
 // TODO: create unit tests for this once mocks for this package are
 // worked out.
 func CacheDependency(a app.App, d pkg.Descriptor, customName string) error {
@@ -80,7 +80,6 @@ func CacheDependency(a app.App, d pkg.Descriptor, customName string) error {
 
 	// Add library to app specification, but wait to write it out until
 	// the end, in case one of the network calls fails.
-
 	log.Infof("Retrieved %d files", len(files))
 
 	for _, dir := range directories {

pkg/registry/cache_test.go

@@ -16,62 +16,36 @@
 package registry
 
 import (
+	"path/filepath"
 	"testing"
 
 	"github.com/ksonnet/ksonnet/metadata/app"
 	amocks "github.com/ksonnet/ksonnet/metadata/app/mocks"
 	"github.com/ksonnet/ksonnet/pkg/pkg"
-	ghutil "github.com/ksonnet/ksonnet/pkg/util/github"
-	"github.com/ksonnet/ksonnet/pkg/util/github/mocks"
+	"github.com/ksonnet/ksonnet/pkg/util/test"
 	"github.com/spf13/afero"
-	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 )
 
 func Test_CacheDependency(t *testing.T) {
 	withApp(t, func(a *amocks.App, fs afero.Fs) {
+		test.StageDir(t, fs, "incubator", filepath.Join("/work", "incubator"))
+
 		libraries := app.LibraryRefSpecs{}
 		a.On("Libraries").Return(libraries, nil)
 
 		registries := app.RegistryRefSpecs{
 			"incubator": &app.RegistryRefSpec{
 				Name:     "incubator",
-				Protocol: ProtocolGitHub,
-				URI:      "github.com/foo/bar/tree/master/incubator",
-				GitVersion: &app.GitVersionSpec{
-					CommitSHA: "54321",
-					RefSpec:   "master",
-				},
+				Protocol: ProtocolFilesystem,
+				URI:      "file:///work/incubator",
 			},
 		}
 		a.On("Registries").Return(registries, nil)
 
-		ghMock := &mocks.GitHub{}
-		ghMock.On("CommitSHA1", mock.Anything, mock.Anything, "master").Return("54321", nil)
-
-		repo := ghutil.Repo{Org: "foo", Repo: "bar"}
-		mockPartFs(t, repo, ghMock, "incubator/apache", "54321")
-
-		registryContent := buildContent(t, registryYAMLFile)
-		ghMock.On(
-			"Contents",
-			mock.Anything,
-			registryYAMLFile,
-			"40285d8a14f1ac5787e405e1023cf0c07f6aa28c").
-			Return(registryContent, nil, nil)
-
-		ghOpt := GitHubClient(ghMock)
-		githubFactory = func(registryRef *app.RegistryRefSpec) (*GitHub, error) {
-			return NewGitHub(registryRef, ghOpt)
-		}
-
 		library := &app.LibraryRefSpec{
 			Name:     "apache",
 			Registry: "incubator",
-			GitVersion: &app.GitVersionSpec{
-				CommitSHA: "54321",
-				RefSpec:   "master",
-			},
 		}
 		a.On("UpdateLib", "apache", library).Return(nil)
 
@@ -79,5 +53,7 @@ func Test_CacheDependency(t *testing.T) {
 
 		err := CacheDependency(a, d, "")
 		require.NoError(t, err)
+
+		test.AssertExists(t, fs, filepath.Join(a.Root(), "vendor", "incubator", "apache", "parts.yaml"))
 	})
 }

pkg/registry/fs.go

@@ -118,6 +118,10 @@ func (fs *Fs) ResolveLibrarySpec(partName, libRefSpec string) (*parts.Spec, erro
 
 // ResolveLibrary fetches the part and creates a parts spec and library ref spec.
 func (fs *Fs) ResolveLibrary(partName, partAlias, libRefSpec string, onFile ResolveFile, onDir ResolveDirectory) (*parts.Spec, *app.LibraryRefSpec, error) {
+	if partAlias == "" {
+		partAlias = partName
+	}
+
 	partRoot := filepath.Join(fs.RegistrySpecDir(), partName)
 	parentDir := filepath.Dir(fs.RegistrySpecDir())
 

pkg/registry/github.go

@@ -20,12 +20,14 @@ import (
 	"fmt"
 	"net/url"
 	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/ksonnet/ksonnet/metadata/app"
 	"github.com/ksonnet/ksonnet/pkg/parts"
 	"github.com/ksonnet/ksonnet/pkg/util/github"
 	"github.com/pkg/errors"
+	"github.com/spf13/afero"
 )
 
 const (
@@ -37,12 +39,12 @@ var (
 	// errInvalidURI is an invalid github uri error.
 	errInvalidURI = fmt.Errorf("Invalid GitHub URI: try navigating in GitHub to the URI of the folder containing the 'yaml', and using that URI instead. Generally, this URI should be of the form 'github.com/{organization}/{repository}/tree/{branch}/[path-to-directory]'")
 
-	githubFactory = func(spec *app.RegistryRefSpec) (*GitHub, error) {
-		return NewGitHub(spec)
+	githubFactory = func(a app.App, spec *app.RegistryRefSpec) (*GitHub, error) {
+		return NewGitHub(a, spec)
 	}
 )
 
-type ghFactoryFn func(spec *app.RegistryRefSpec) (*GitHub, error)
+type ghFactoryFn func(a app.App, spec *app.RegistryRefSpec) (*GitHub, error)
 
 // GitHubClient is an option for the setting a github client.
 func GitHubClient(c github.GitHub) GitHubOpt {
@@ -56,6 +58,7 @@ type GitHubOpt func(*GitHub)
 
 // GitHub is a Github Registry
 type GitHub struct {
+	app      app.App
 	name     string
 	hd       *hubDescriptor
 	ghClient github.GitHub
@@ -63,12 +66,13 @@ type GitHub struct {
 }
 
 // NewGitHub creates an instance of GitHub.
-func NewGitHub(registryRef *app.RegistryRefSpec, opts ...GitHubOpt) (*GitHub, error) {
+func NewGitHub(a app.App, registryRef *app.RegistryRefSpec, opts ...GitHubOpt) (*GitHub, error) {
 	if registryRef == nil {
 		return nil, errors.New("registry ref is nil")
 	}
 
 	gh := &GitHub{
+		app:      a,
 		name:     registryRef.Name,
 		spec:     registryRef,
 		ghClient: github.DefaultClient,
@@ -80,19 +84,21 @@ func NewGitHub(registryRef *app.RegistryRefSpec, opts ...GitHubOpt) (*GitHub, er
 	}
 	gh.hd = hd
 
-	for _, opt := range opts {
-		opt(gh)
-	}
+	if gh.spec.GitVersion == nil || gh.spec.GitVersion.CommitSHA == "" {
+		for _, opt := range opts {
+			opt(gh)
+		}
 
-	ctx := context.Background()
-	sha, err := gh.ghClient.CommitSHA1(ctx, hd.Repo(), hd.refSpec)
-	if err != nil {
-		return nil, errors.Wrap(err, "unable to find SHA1 for repo")
-	}
+		ctx := context.Background()
+		sha, err := gh.ghClient.CommitSHA1(ctx, hd.Repo(), hd.refSpec)
+		if err != nil {
+			return nil, errors.Wrap(err, "unable to find SHA1 for repo")
+		}
 
-	gh.spec.GitVersion = &app.GitVersionSpec{
-		RefSpec:   hd.refSpec,
-		CommitSHA: sha,
+		gh.spec.GitVersion = &app.GitVersionSpec{
+			RefSpec:   hd.refSpec,
+			CommitSHA: sha,
+		}
 	}
 
 	return gh, nil
@@ -133,6 +139,47 @@ func (gh *GitHub) RegistrySpecFilePath() string {
 
 // FetchRegistrySpec fetches the registry spec.
 func (gh *GitHub) FetchRegistrySpec() (*Spec, error) {
+	// Check local disk cache.
+	registrySpecFile := makePath(gh.app, gh)
+	registrySpec, exists, err := load(gh.app, registrySpecFile)
+	if err != nil {
+		return nil, errors.Wrap(err, "load registry spec file")
+	}
+
+	if !exists {
+		// If failed, use the protocol to try to retrieve app specification.
+		registrySpec, err = gh.cacheRegistrySpec()
+		if err != nil {
+			return nil, err
+		}
+
+		var registrySpecBytes []byte
+		registrySpecBytes, err = registrySpec.Marshal()
+		if err != nil {
+			return nil, err
+		}
+
+		// NOTE: We call mkdir after getting the registry spec, since a
+		// network call might fail and leave this half-initialized empty
+		// directory.
+		registrySpecDir := filepath.Join(root(gh.app), gh.RegistrySpecDir())
+		err = gh.app.Fs().MkdirAll(registrySpecDir, app.DefaultFolderPermissions)
+		if err != nil {
+			return nil, err
+		}
+
+		err = afero.WriteFile(gh.app.Fs(), registrySpecFile, registrySpecBytes, app.DefaultFilePermissions)
+		if err != nil {
+			return nil, err
+		}
+	} else if err != nil {
+		return nil, err
+	}
+
+	return registrySpec, nil
+}
+
+func (gh *GitHub) cacheRegistrySpec() (*Spec, error) {
 	ctx := context.Background()
 
 	file, _, err := gh.ghClient.Contents(ctx, gh.hd.Repo(), gh.hd.regSpecRepoPath,

pkg/registry/github_test.go

@@ -24,15 +24,23 @@ import (
 
 	"github.com/google/go-github/github"
 	"github.com/ksonnet/ksonnet/metadata/app"
+	amocks "github.com/ksonnet/ksonnet/metadata/app/mocks"
 	"github.com/ksonnet/ksonnet/pkg/parts"
 	ghutil "github.com/ksonnet/ksonnet/pkg/util/github"
 	"github.com/ksonnet/ksonnet/pkg/util/github/mocks"
+	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 )
 
 func makeGh(t *testing.T, u, sha1 string) (*GitHub, *mocks.GitHub) {
+	fs := afero.NewMemMapFs()
+	appMock := &amocks.App{}
+	appMock.On("Fs").Return(fs)
+	appMock.On("Root").Return("/app")
+	appMock.On("LibPath", mock.AnythingOfType("string")).Return(filepath.Join("/app", "lib", "v1.8.7"), nil)
+
 	ghMock := &mocks.GitHub{}
 	ghMock.On("CommitSHA1", mock.Anything, ghutil.Repo{Org: "ksonnet", Repo: "parts"}, "master").
 		Return(sha1, nil)
@@ -45,7 +53,7 @@ func makeGh(t *testing.T, u, sha1 string) (*GitHub, *mocks.GitHub) {
 		URI:      "github.com/ksonnet/parts/tree/master/incubator",
 	}
 
-	g, err := NewGitHub(spec, optGh)
+	g, err := NewGitHub(appMock, spec, optGh)
 	require.NoError(t, err)
 
 	return g, ghMock

pkg/registry/manager.go

@@ -57,7 +57,7 @@ func Package(a app.App, name string) (*pkg.Package, error) {
 func Locate(a app.App, spec *app.RegistryRefSpec) (Registry, error) {
 	switch spec.Protocol {
 	case ProtocolGitHub:
-		return githubFactory(spec)
+		return githubFactory(a, spec)
 	case ProtocolFilesystem:
 		return NewFs(a, spec)
 	default:

pkg/registry/manager_test.go

@@ -38,8 +38,8 @@ func Test_Package(t *testing.T) {
 			Return(content, nil, nil)
 
 		ghcOpt := GitHubClient(c)
-		githubFactory = func(spec *app.RegistryRefSpec) (*GitHub, error) {
-			return NewGitHub(spec, ghcOpt)
+		githubFactory = func(a app.App, spec *app.RegistryRefSpec) (*GitHub, error) {
+			return NewGitHub(a, spec, ghcOpt)
 		}
 
 		registries := app.RegistryRefSpecs{
@@ -66,8 +66,8 @@ func Test_List(t *testing.T) {
 			Return("12345", nil)
 
 		ghcOpt := GitHubClient(c)
-		githubFactory = func(spec *app.RegistryRefSpec) (*GitHub, error) {
-			return NewGitHub(spec, ghcOpt)
+		githubFactory = func(a app.App, spec *app.RegistryRefSpec) (*GitHub, error) {
+			return NewGitHub(a, spec, ghcOpt)
 		}
 
 		specs := app.RegistryRefSpecs{