Unverified Commit e938e5fe authored by Bryan Liles's avatar Bryan Liles Committed by GitHub
Browse files

Merge pull request #406 from bryanl/github-registry-cache

Allow for regeneration of lib and registry cache
parents 2d83a315 5c48e9c1
......@@ -37,8 +37,11 @@ func RunInit(m map[string]interface{}) error {
return i.Run()
}
type appLoadFn func(fs afero.Fs, root string, skipFindRoot bool) (app.App, error)
type appInitFn func(fs afero.Fs, name, rootPath, k8sSpecFlag, serverURI, namespace string, registries []registry.Registry) error
type initIncubatorFn func() (registry.Registry, error)
type initIncubatorFn func(app.App) (registry.Registry, error)
// Init creates a component namespace
type Init struct {
......@@ -51,6 +54,7 @@ type Init struct {
skipDefaultRegistries bool
appInitFn appInitFn
appLoadFn appLoadFn
initIncubatorFn initIncubatorFn
}
......@@ -68,6 +72,7 @@ func NewInit(m map[string]interface{}) (*Init, error) {
skipDefaultRegistries: ol.loadBool(OptionSkipDefaultRegistries),
appInitFn: appinit.Init,
appLoadFn: app.Load,
initIncubatorFn: initIncubator,
}
......@@ -83,7 +88,12 @@ func (i *Init) Run() error {
var registries []registry.Registry
if !i.skipDefaultRegistries {
gh, err := i.initIncubatorFn()
a, err := i.appLoadFn(i.fs, i.rootPath, true)
if err != nil {
return err
}
gh, err := i.initIncubatorFn(a)
if err != nil {
return err
}
......@@ -102,10 +112,12 @@ func (i *Init) Run() error {
)
}
func initIncubator() (registry.Registry, error) {
return registry.NewGitHub(&app.RegistryRefSpec{
Name: "incubator",
Protocol: registry.ProtocolGitHub,
URI: defaultIncubatorURI,
})
func initIncubator(a app.App) (registry.Registry, error) {
return registry.NewGitHub(
a,
&app.RegistryRefSpec{
Name: "incubator",
Protocol: registry.ProtocolGitHub,
URI: defaultIncubatorURI,
})
}
......@@ -18,6 +18,7 @@ package actions
import (
"testing"
"github.com/ksonnet/ksonnet/metadata/app"
amocks "github.com/ksonnet/ksonnet/metadata/app/mocks"
"github.com/ksonnet/ksonnet/pkg/registry"
rmocks "github.com/ksonnet/ksonnet/pkg/registry/mocks"
......@@ -85,7 +86,11 @@ func TestInit(t *testing.T) {
return nil
}
a.initIncubatorFn = func() (registry.Registry, error) {
a.appLoadFn = func(fs afero.Fs, root string, skipFindRoot bool) (app.App, error) {
return appMock, nil
}
a.initIncubatorFn = func(a app.App) (registry.Registry, error) {
r := &rmocks.Registry{}
r.On("Protocol").Return("github")
r.On("URI").Return("github.com/ksonnet/parts/tree/master/incubator")
......
......@@ -102,7 +102,7 @@ application configuration to remote clusters.
isInit = true
}
ka, err = app.Load(appFs, wd)
ka, err = app.Load(appFs, wd, false)
if err != nil && isInit {
return err
}
......
// Copyright 2018 The ksonnet authors
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build e2e
package e2e
......@@ -22,6 +37,10 @@ var _ = Describe("ks init", func() {
o := a.registryList()
assertOutput(filepath.Join("init", "registry-output.txt"), o.stdout)
})
It("creates a gitignore", func() {
assertContents(filepath.Join("init", "gitignore"), filepath.Join(a.dir, ".gitignore"))
})
})
Context("without default registries", func() {
......
......@@ -18,6 +18,7 @@
package e2e
import (
"os"
"path/filepath"
. "github.com/onsi/ginkgo"
......@@ -101,6 +102,19 @@ var _ = Describe("ks pkg", func() {
assertExitStatus(o, 0)
assertOutput("pkg/list/output.txt", o.stdout)
})
Context("git spec cache has been been deleted", func() {
JustBeforeEach(func() {
err := os.RemoveAll(filepath.Join(a.dir, ".ksonnet"))
Expect(err).NotTo(HaveOccurred())
})
It("generates spec cache", func() {
o := a.runKs("pkg", "list")
assertExitStatus(o, 0)
assertOutput("pkg/list/output.txt", o.stdout)
})
})
})
Context("use", func() {
......
......@@ -18,7 +18,11 @@
package e2e
import (
"os"
"path/filepath"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("ks show", func() {
......@@ -35,4 +39,17 @@ var _ = Describe("ks show", func() {
assertOutput("show/output.txt", o.stdout)
})
Context("lib does not exists", func() {
JustBeforeEach(func() {
err := os.RemoveAll(filepath.Join(a.dir, "lib"))
Expect(err).NotTo(HaveOccurred())
})
It("generates spec cache", func() {
o := a.runKs("show", "default")
assertExitStatus(o, 0)
assertOutput("show/output.txt", o.stdout)
})
})
})
/lib
/.ksonnet/registries
......@@ -71,15 +71,19 @@ type App interface {
}
// Load loads the application configuration.
func Load(fs afero.Fs, cwd string) (App, error) {
appRoot, err := findRoot(fs, cwd)
if err != nil {
return nil, err
func Load(fs afero.Fs, cwd string, skipFindRoot bool) (App, error) {
appRoot := cwd
if !skipFindRoot {
var err error
appRoot, err = findRoot(fs, cwd)
if err != nil {
return nil, err
}
}
spec, err := read(fs, appRoot)
if err != nil {
return nil, err
return NewApp010(fs, appRoot), nil
}
switch spec.APIVersion {
......
......@@ -146,7 +146,7 @@ func (m *manager) Root() string {
}
func (m *manager) App() (app.App, error) {
return app.Load(m.appFS, m.rootPath)
return app.Load(m.appFS, m.rootPath, false)
}
func (m *manager) LibPaths() (envPath, vendorPath string) {
......
......@@ -65,7 +65,7 @@ func (i *initApp) Run() error {
}
// Load application.
a, err := app.Load(i.fs, i.rootPath)
a, err := app.Load(i.fs, i.rootPath, false)
if err != nil {
return err
}
......@@ -187,6 +187,10 @@ func (i *initApp) createAppDirTree() error {
path string
content []byte
}{
{
filepath.Join(i.rootPath, ".gitignore"),
ignoreData,
},
{
filepath.Join(i.rootPath, "components", "params.libsonnet"),
component.GenParamsContent(),
......@@ -210,3 +214,7 @@ func (i *initApp) createAppDirTree() error {
return nil
}
var ignoreData = []byte(`/lib
/.ksonnet/registries
`)
......@@ -94,6 +94,7 @@ func TestInit(t *testing.T) {
func checkApp(t *testing.T, fs afero.Fs, rootPath, version, namespace string) {
expectedDirs := []string{
".gitignore",
"app.yaml",
filepath.Join(".ksonnet", "registries", "testdata", "registry.yaml"),
filepath.Join("components", "params.libsonnet"),
......
......@@ -16,8 +16,6 @@
package registry
import (
"path/filepath"
"github.com/ksonnet/ksonnet/metadata/app"
"github.com/pkg/errors"
"github.com/spf13/afero"
......@@ -37,7 +35,7 @@ func Add(a app.App, name, protocol, uri, version string, isOverride bool) (*Spec
switch protocol {
case ProtocolGitHub:
r, err = githubFactory(initSpec)
r, err = githubFactory(a, initSpec)
case ProtocolFilesystem:
r, err = NewFs(a, initSpec)
default:
......@@ -54,7 +52,7 @@ func Add(a app.App, name, protocol, uri, version string, isOverride bool) (*Spec
}
// Retrieve the contents of registry.
registrySpec, err := getOrCacheRegistry(a, r)
registrySpec, err := r.FetchRegistrySpec()
if err != nil {
return nil, errors.Wrap(err, "cache registry")
}
......@@ -62,46 +60,6 @@ func Add(a app.App, name, protocol, uri, version string, isOverride bool) (*Spec
return registrySpec, nil
}
func getOrCacheRegistry(a app.App, gh Registry) (*Spec, error) {
// Check local disk cache.
registrySpecFile := makePath(a, gh)
registrySpec, exists, err := load(a, registrySpecFile)
if err != nil {
return nil, errors.Wrap(err, "load registry spec file")
}
if !exists {
// If failed, use the protocol to try to retrieve app specification.
registrySpec, err = gh.FetchRegistrySpec()
if err != nil {
return nil, err
}
registrySpecBytes, err := registrySpec.Marshal()
if err != nil {
return nil, err
}
// NOTE: We call mkdir after getting the registry spec, since a
// network call might fail and leave this half-initialized empty
// directory.
registrySpecDir := filepath.Join(root(a), gh.RegistrySpecDir())
err = a.Fs().MkdirAll(registrySpecDir, app.DefaultFolderPermissions)
if err != nil {
return nil, err
}
err = afero.WriteFile(a.Fs(), registrySpecFile, registrySpecBytes, app.DefaultFilePermissions)
if err != nil {
return nil, err
}
} else if err != nil {
return nil, err
}
return registrySpec, nil
}
func load(a app.App, path string) (*Spec, bool, error) {
exists, err := afero.Exists(a.Fs(), path)
if err != nil {
......
......@@ -69,8 +69,8 @@ func TestAdd(t *testing.T) {
Return(registryContent, nil, nil)
ghOpt := GitHubClient(ghMock)
githubFactory = func(registryRef *app.RegistryRefSpec) (*GitHub, error) {
return NewGitHub(registryRef, ghOpt)
githubFactory = func(a app.App, registryRef *app.RegistryRefSpec) (*GitHub, error) {
return NewGitHub(a, registryRef, ghOpt)
}
spec, err := Add(appMock, "new", ProtocolGitHub, "github.com/foo/bar", "", true)
......
......@@ -26,7 +26,7 @@ import (
"github.com/spf13/afero"
)
// CacheDependency caches registry dependencies.
// CacheDependency vendors registry dependencies.
// TODO: create unit tests for this once mocks for this package are
// worked out.
func CacheDependency(a app.App, d pkg.Descriptor, customName string) error {
......@@ -80,7 +80,6 @@ func CacheDependency(a app.App, d pkg.Descriptor, customName string) error {
// Add library to app specification, but wait to write it out until
// the end, in case one of the network calls fails.
log.Infof("Retrieved %d files", len(files))
for _, dir := range directories {
......
......@@ -16,62 +16,36 @@
package registry
import (
"path/filepath"
"testing"
"github.com/ksonnet/ksonnet/metadata/app"
amocks "github.com/ksonnet/ksonnet/metadata/app/mocks"
"github.com/ksonnet/ksonnet/pkg/pkg"
ghutil "github.com/ksonnet/ksonnet/pkg/util/github"
"github.com/ksonnet/ksonnet/pkg/util/github/mocks"
"github.com/ksonnet/ksonnet/pkg/util/test"
"github.com/spf13/afero"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
func Test_CacheDependency(t *testing.T) {
withApp(t, func(a *amocks.App, fs afero.Fs) {
test.StageDir(t, fs, "incubator", filepath.Join("/work", "incubator"))
libraries := app.LibraryRefSpecs{}
a.On("Libraries").Return(libraries, nil)
registries := app.RegistryRefSpecs{
"incubator": &app.RegistryRefSpec{
Name: "incubator",
Protocol: ProtocolGitHub,
URI: "github.com/foo/bar/tree/master/incubator",
GitVersion: &app.GitVersionSpec{
CommitSHA: "54321",
RefSpec: "master",
},
Protocol: ProtocolFilesystem,
URI: "file:///work/incubator",
},
}
a.On("Registries").Return(registries, nil)
ghMock := &mocks.GitHub{}
ghMock.On("CommitSHA1", mock.Anything, mock.Anything, "master").Return("54321", nil)
repo := ghutil.Repo{Org: "foo", Repo: "bar"}
mockPartFs(t, repo, ghMock, "incubator/apache", "54321")
registryContent := buildContent(t, registryYAMLFile)
ghMock.On(
"Contents",
mock.Anything,
registryYAMLFile,
"40285d8a14f1ac5787e405e1023cf0c07f6aa28c").
Return(registryContent, nil, nil)
ghOpt := GitHubClient(ghMock)
githubFactory = func(registryRef *app.RegistryRefSpec) (*GitHub, error) {
return NewGitHub(registryRef, ghOpt)
}
library := &app.LibraryRefSpec{
Name: "apache",
Registry: "incubator",
GitVersion: &app.GitVersionSpec{
CommitSHA: "54321",
RefSpec: "master",
},
}
a.On("UpdateLib", "apache", library).Return(nil)
......@@ -79,5 +53,7 @@ func Test_CacheDependency(t *testing.T) {
err := CacheDependency(a, d, "")
require.NoError(t, err)
test.AssertExists(t, fs, filepath.Join(a.Root(), "vendor", "incubator", "apache", "parts.yaml"))
})
}
......@@ -118,6 +118,10 @@ func (fs *Fs) ResolveLibrarySpec(partName, libRefSpec string) (*parts.Spec, erro
// ResolveLibrary fetches the part and creates a parts spec and library ref spec.
func (fs *Fs) ResolveLibrary(partName, partAlias, libRefSpec string, onFile ResolveFile, onDir ResolveDirectory) (*parts.Spec, *app.LibraryRefSpec, error) {
if partAlias == "" {
partAlias = partName
}
partRoot := filepath.Join(fs.RegistrySpecDir(), partName)
parentDir := filepath.Dir(fs.RegistrySpecDir())
......
......@@ -20,12 +20,14 @@ import (
"fmt"
"net/url"
"path"
"path/filepath"
"strings"
"github.com/ksonnet/ksonnet/metadata/app"
"github.com/ksonnet/ksonnet/pkg/parts"
"github.com/ksonnet/ksonnet/pkg/util/github"
"github.com/pkg/errors"
"github.com/spf13/afero"
)
const (
......@@ -37,12 +39,12 @@ var (
// errInvalidURI is an invalid github uri error.
errInvalidURI = fmt.Errorf("Invalid GitHub URI: try navigating in GitHub to the URI of the folder containing the 'yaml', and using that URI instead. Generally, this URI should be of the form 'github.com/{organization}/{repository}/tree/{branch}/[path-to-directory]'")
githubFactory = func(spec *app.RegistryRefSpec) (*GitHub, error) {
return NewGitHub(spec)
githubFactory = func(a app.App, spec *app.RegistryRefSpec) (*GitHub, error) {
return NewGitHub(a, spec)
}
)
type ghFactoryFn func(spec *app.RegistryRefSpec) (*GitHub, error)
type ghFactoryFn func(a app.App, spec *app.RegistryRefSpec) (*GitHub, error)
// GitHubClient is an option for the setting a github client.
func GitHubClient(c github.GitHub) GitHubOpt {
......@@ -56,6 +58,7 @@ type GitHubOpt func(*GitHub)
// GitHub is a Github Registry
type GitHub struct {
app app.App
name string
hd *hubDescriptor
ghClient github.GitHub
......@@ -63,12 +66,13 @@ type GitHub struct {
}
// NewGitHub creates an instance of GitHub.
func NewGitHub(registryRef *app.RegistryRefSpec, opts ...GitHubOpt) (*GitHub, error) {
func NewGitHub(a app.App, registryRef *app.RegistryRefSpec, opts ...GitHubOpt) (*GitHub, error) {
if registryRef == nil {
return nil, errors.New("registry ref is nil")
}
gh := &GitHub{
app: a,
name: registryRef.Name,
spec: registryRef,
ghClient: github.DefaultClient,
......@@ -80,19 +84,21 @@ func NewGitHub(registryRef *app.RegistryRefSpec, opts ...GitHubOpt) (*GitHub, er
}
gh.hd = hd
for _, opt := range opts {
opt(gh)
}
if gh.spec.GitVersion == nil || gh.spec.GitVersion.CommitSHA == "" {
for _, opt := range opts {
opt(gh)
}
ctx := context.Background()
sha, err := gh.ghClient.CommitSHA1(ctx, hd.Repo(), hd.refSpec)
if err != nil {
return nil, errors.Wrap(err, "unable to find SHA1 for repo")
}
ctx := context.Background()
sha, err := gh.ghClient.CommitSHA1(ctx, hd.Repo(), hd.refSpec)
if err != nil {
return nil, errors.Wrap(err, "unable to find SHA1 for repo")
}
gh.spec.GitVersion = &app.GitVersionSpec{
RefSpec: hd.refSpec,
CommitSHA: sha,
gh.spec.GitVersion = &app.GitVersionSpec{
RefSpec: hd.refSpec,
CommitSHA: sha,
}
}
return gh, nil
......@@ -133,6 +139,47 @@ func (gh *GitHub) RegistrySpecFilePath() string {
// FetchRegistrySpec fetches the registry spec.
func (gh *GitHub) FetchRegistrySpec() (*Spec, error) {
// Check local disk cache.
registrySpecFile := makePath(gh.app, gh)
registrySpec, exists, err := load(gh.app, registrySpecFile)
if err != nil {
return nil, errors.Wrap(err, "load registry spec file")
}
if !exists {
// If failed, use the protocol to try to retrieve app specification.
registrySpec, err = gh.cacheRegistrySpec()
if err != nil {
return nil, err
}
var registrySpecBytes []byte
registrySpecBytes, err = registrySpec.Marshal()
if err != nil {
return nil, err
}
// NOTE: We call mkdir after getting the registry spec, since a
// network call might fail and leave this half-initialized empty
// directory.
registrySpecDir := filepath.Join(root(gh.app), gh.RegistrySpecDir())
err = gh.app.Fs().MkdirAll(registrySpecDir, app.DefaultFolderPermissions)
if err != nil {
return nil, err
}
err = afero.WriteFile(gh.app.Fs(), registrySpecFile, registrySpecBytes, app.DefaultFilePermissions)
if err != nil {
return nil, err
}
} else if err != nil {
return nil, err
}
return registrySpec, nil
}
func (gh *GitHub) cacheRegistrySpec() (*Spec, error) {
ctx := context.Background()
file, _, err := gh.ghClient.Contents(ctx, gh.hd.Repo(), gh.hd.regSpecRepoPath,
......
......@@ -24,15 +24,23 @@ import (
"github.com/google/go-github/github"
"github.com/ksonnet/ksonnet/metadata/app"
amocks "github.com/ksonnet/ksonnet/metadata/app/mocks"
"github.com/ksonnet/ksonnet/pkg/parts"
ghutil "github.com/ksonnet/ksonnet/pkg/util/github"
"github.com/ksonnet/ksonnet/pkg/util/github/mocks"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
func makeGh(t *testing.T, u, sha1 string) (*GitHub, *mocks.GitHub) {
fs := afero.NewMemMapFs()
appMock := &amocks.App{}
appMock.On("Fs").Return(fs)
appMock.On("Root").Return("/app")
appMock.On("LibPath", mock.AnythingOfType("string")).Return(filepath.Join("/app", "lib", "v1.8.7"), nil)
ghMock := &mocks.GitHub{}
ghMock.On("CommitSHA1", mock.Anything, ghutil.Repo{Org: "ksonnet", Repo: "parts"}, "master").
Return(sha1, nil)
......@@ -45,7 +53,7 @@ func makeGh(t *testing.T, u, sha1 string) (*GitHub, *mocks.GitHub) {
URI: "github.com/ksonnet/parts/tree/master/incubator",
}
g, err := NewGitHub(spec, optGh)
g, err := NewGitHub(appMock, spec, optGh)
require.NoError(t, err)
return g, ghMock
......
......@@ -57,7 +57,7 @@ func Package(a app.App, name string) (*pkg.Package, error) {
func Locate(a app.App, spec *app.RegistryRefSpec) (Registry, error) {
switch spec.Protocol {
case ProtocolGitHub: