Add ability to fetch dependencies and manage registries

metadata/app/schema.go

@@ -27,7 +27,7 @@ const (
 )
 
 var ErrRegistryNameInvalid = fmt.Errorf("Registry name is invalid")
-var ErrRegistryExists = fmt.Errorf("Registry with name already exists, but has different URL or protocol")
+var ErrRegistryExists = fmt.Errorf("Registry with name already exists")
 
 type Spec struct {
 	APIVersion   string           `json:"apiVersion,omitempty"`
@@ -41,6 +41,7 @@ type Spec struct {
 	Bugs         string           `json:"bugs,omitempty"`
 	Keywords     []string         `json:"keywords,omitempty"`
 	Registries   RegistryRefSpecs `json:"registries,omitempty"`
+	Libraries    LibraryRefSpecs  `json:"libraries,omitempty"`
 	License      string           `json:"license,omitempty"`
 }
 
@@ -54,35 +55,22 @@ func (s *Spec) GetRegistryRef(name string) (*RegistryRefSpec, bool) {
 		// Populate name, which we do not include in the deserialization
 		// process.
 		registryRefSpec.Name = name
-		return registryRefSpec, true
 	}
-
-	return nil, false
+	return registryRefSpec, ok
 }
 
-func (s *Spec) AddRegistryRef(name, protocol, uri string) (*RegistryRefSpec, error) {
-	if name == "" {
-		return nil, ErrRegistryNameInvalid
+func (s *Spec) AddRegistryRef(registryRefSpec *RegistryRefSpec) error {
+	if registryRefSpec.Name == "" {
+		return ErrRegistryNameInvalid
 	}
 
-	registryRefSpec, registryRefExists := s.Registries[name]
+	_, registryRefExists := s.Registries[registryRefSpec.Name]
 	if registryRefExists {
-		storedURI, uriExists := registryRefSpec.Spec["uri"]
-		if registryRefSpec.Protocol != protocol || !uriExists || storedURI != uri {
-			return nil, ErrRegistryExists
-		}
-	} else {
-		registryRefSpec = &RegistryRefSpec{
-			Protocol: protocol,
-			Spec: map[string]interface{}{
-				"uri": uri,
-			},
-			Name: name,
-		}
+		return ErrRegistryExists
 	}
 
-	s.Registries[name] = registryRefSpec
-	return registryRefSpec, nil
+	s.Registries[registryRefSpec.Name] = registryRefSpec
+	return nil
 }
 
 type RepositorySpec struct {
@@ -91,13 +79,26 @@ type RepositorySpec struct {
 }
 
 type RegistryRefSpec struct {
-	Protocol string                 `json:"protocol"`
-	Spec     map[string]interface{} `json:"spec"`
-	Name     string
+	Name       string
+	Protocol   string          `json:"protocol"`
+	URI        string          `json:"uri"`
+	GitVersion *GitVersionSpec `json:"gitVersion"`
 }
 
 type RegistryRefSpecs map[string]*RegistryRefSpec
 
+type LibraryRefSpec struct {
+	Name       string          `json:"name"`
+	GitVersion *GitVersionSpec `json:"gitVersion"`
+}
+
+type GitVersionSpec struct {
+	RefSpec   string `json:"refSpec"`
+	CommitSHA string `json:"commitSha"`
+}
+
+type LibraryRefSpecs map[string]*LibraryRefSpec
+
 type ContributorSpec struct {
 	Name  string `json:"name"`
 	Email string `json:"email"`

metadata/app/schema_test.go

+// Copyright 2017 The kubecfg authors
+//
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+
 package app
 
 import (
+	"fmt"
 	"testing"
 )
 
+func makeSimpleRefSpec(name, protocol, uri, version string) *RegistryRefSpec {
+	return &RegistryRefSpec{
+		Name:     name,
+		Protocol: protocol,
+		URI:      uri,
+		GitVersion: &GitVersionSpec{
+			RefSpec:   version,
+			CommitSHA: version,
+		},
+	}
+}
+
 func TestGetRegistryRefSuccess(t *testing.T) {
 	example1 := Spec{
 		Registries: RegistryRefSpecs{
 			"simple1": &RegistryRefSpec{
-				Spec: map[string]interface{}{
-					"uri": "example.com",
-				},
+				URI:      "example.com",
 				Protocol: "github",
 			},
 		},
 	}
 
 	r1, ok := example1.GetRegistryRef("simple1")
+	fmt.Println(r1)
 	if r1 == nil || !ok {
 		t.Error("Expected registry to contain 'simple1'")
 	}
 
-	uri, ok := r1.Spec["uri"]
-	if !ok || uri.(string) != "example.com" || r1.Name != "simple1" || r1.Protocol != "github" {
+	if r1.URI != "example.com" || r1.Name != "simple1" || r1.Protocol != "github" {
 		t.Errorf("Registry did not add correct values:\n%s", r1)
 	}
 }
@@ -31,9 +57,7 @@ func TestGetRegistryRefFailure(t *testing.T) {
 	example1 := Spec{
 		Registries: RegistryRefSpecs{
 			"simple1": &RegistryRefSpec{
-				Spec: map[string]interface{}{
-					"uri": "example.com",
-				},
+				URI:      "example.com",
 				Protocol: "github",
 			},
 		},
@@ -50,24 +74,18 @@ func TestAddRegistryRefSuccess(t *testing.T) {
 		Registries: RegistryRefSpecs{},
 	}
 
-	r1, err := example1.AddRegistryRef("simple1", "github", "example.com")
-	if r1 == nil || err != nil {
+	err := example1.AddRegistryRef(makeSimpleRefSpec("simple1", "github", "example.com", "master"))
+	if err != nil {
 		t.Errorf("Expected registry add to succeed:\n%s", err)
 	}
 
-	uri1, ok1 := r1.Spec["uri"]
-	if !ok1 || uri1.(string) != "example.com" || r1.Name != "simple1" || r1.Protocol != "github" {
+	r1, ok1 := example1.GetRegistryRef("simple1")
+	if !ok1 || r1.URI != "example.com" || r1.Name != "simple1" || r1.Protocol != "github" {
 		t.Errorf("Registry did not add correct values:\n%s", r1)
 	}
 
-	// Test that the `Name` field is added properly if it already exists.
-	r2, err := example1.AddRegistryRef("simple1", "github", "example.com")
-	if r2 == nil || err != nil {
-		t.Errorf("Expected registry add to succeed:\n%s", err)
-	}
-
-	uri2, ok2 := r2.Spec["uri"]
-	if !ok2 || uri2.(string) != "example.com" || r1.Name != "simple1" || r1.Protocol != "github" {
+	r2, ok2 := example1.GetRegistryRef("simple1")
+	if !ok2 || r2.URI != "example.com" || r2.Name != "simple1" || r2.Protocol != "github" {
 		t.Errorf("Registry did not add correct values:\n%s", r1)
 	}
 }
@@ -76,26 +94,24 @@ func TestAddRegistryRefFailure(t *testing.T) {
 	example1 := Spec{
 		Registries: RegistryRefSpecs{
 			"simple1": &RegistryRefSpec{
-				Spec: map[string]interface{}{
-					"uri": "example.com",
-				},
+				URI:      "example.com",
 				Protocol: "github",
 			},
 		},
 	}
 
-	r1, err := example1.AddRegistryRef("", "github", "example.com")
-	if r1 != nil || err != ErrRegistryNameInvalid {
+	err := example1.AddRegistryRef(makeSimpleRefSpec("", "github", "example.com", "master"))
+	if err != ErrRegistryNameInvalid {
 		t.Error("Expected registry to fail to add registry with invalid name")
 	}
 
-	r2, err := example1.AddRegistryRef("simple1", "fakeProtocol", "example.com")
-	if r2 != nil || err != ErrRegistryExists {
+	err = example1.AddRegistryRef(makeSimpleRefSpec("simple1", "fakeProtocol", "example.com", "master"))
+	if err != ErrRegistryExists {
 		t.Error("Expected registry to fail to add registry with duplicate name and different protocol")
 	}
 
-	r3, err := example1.AddRegistryRef("simple1", "github", "fakeUrl")
-	if r3 != nil || err != ErrRegistryExists {
+	err = example1.AddRegistryRef(makeSimpleRefSpec("simple1", "github", "fakeUrl", "master"))
+	if err != ErrRegistryExists {
 		t.Error("Expected registry to fail to add registry with duplicate name and different uri")
 	}
 }

metadata/interface.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/ksonnet/ksonnet/metadata/app"
 	param "github.com/ksonnet/ksonnet/metadata/params"
+	"github.com/ksonnet/ksonnet/metadata/parts"
 	"github.com/ksonnet/ksonnet/metadata/registry"
 	"github.com/ksonnet/ksonnet/prototype"
 	"github.com/spf13/afero"
@@ -72,7 +73,9 @@ type Manager interface {
 	AppSpec() (*app.Spec, error)
 
 	// Registry API.
-	AddRegistry(name, protocol, uri string) (*registry.Spec, error)
+	AddRegistry(name, protocol, uri, version string) (*registry.Spec, error)
+	GetRegistry(name string) (*registry.Spec, string, error)
+	CacheDependency(registryName, libID, libName, libVersion string) (*parts.Spec, error)
 }
 
 // Find will recursively search the current directory and its parents for a
@@ -91,16 +94,12 @@ func Init(name string, rootPath AbsPath, spec ClusterSpec, serverURI, namespace
 	const (
 		defaultIncubatorRegName = "incubator"
 		defaultIncubatorURI     = "github.com/ksonnet/parts/tree/test-reg/" + defaultIncubatorRegName
-		defaultIncubatorRefSpec = "test-reg"
 	)
 
 	gh, err := makeGitHubRegistryManager(&app.RegistryRefSpec{
+		Name:     "incubator",
 		Protocol: "github",
-		Spec: map[string]interface{}{
-			uriField:     defaultIncubatorURI,
-			refSpecField: defaultIncubatorRefSpec,
-		},
-		Name: "incubator",
+		URI:      defaultIncubatorURI,
 	})
 	if err != nil {
 		return nil, err

metadata/manager.go

@@ -50,6 +50,7 @@ const (
 	baseLibsonnetFile   = "base.libsonnet"
 	appYAMLFile         = "app.yaml"
 	registryYAMLFile    = "registry.yaml"
+	partsYAMLFile       = "parts.yaml"
 
 	// ComponentsExtCodeKey is the ExtCode key for component imports
 	ComponentsExtCodeKey = "__ksonnet/components"
@@ -123,18 +124,23 @@ func initManager(name string, rootPath AbsPath, spec ClusterSpec, serverURI, nam
 		return nil, err
 	}
 
-	regSpec, err := incubatorReg.FindSpec()
+	// Retrieve `registry.yaml`.
+	registryYAMLData, err := generateRegistryYAMLData(incubatorReg)
 	if err != nil {
 		return nil, err
 	}
 
-	registrySpecBytes, err := regSpec.Marshal()
+	// Generate data for `app.yaml`.
+	appYAMLData, err := generateAppYAMLData(name, incubatorReg.MakeRegistryRefSpec())
 	if err != nil {
 		return nil, err
 	}
 
+	// Generate data for `base.libsonnet`.
+	baseLibData := genBaseLibsonnetContent()
+
 	// Initialize directory structure.
-	if err := m.createAppDirTree(name, incubatorReg); err != nil {
+	if err := m.createAppDirTree(name, appYAMLData, baseLibData, incubatorReg); err != nil {
 		return nil, err
 	}
 
@@ -152,7 +158,8 @@ func initManager(name string, rootPath AbsPath, spec ClusterSpec, serverURI, nam
 	}
 
 	// Write out `incubator` registry spec.
-	err = afero.WriteFile(m.appFS, string(m.registryPath(incubatorReg)), registrySpecBytes, defaultFilePermissions)
+	registryPath := string(m.registryPath(incubatorReg))
+	err = afero.WriteFile(m.appFS, registryPath, registryYAMLData, defaultFilePermissions)
 	if err != nil {
 		return nil, err
 	}
@@ -297,6 +304,18 @@ func (m *manager) AppSpec() (*app.Spec, error) {
 		return nil, err
 	}
 
+	if schema.Contributors == nil {
+		schema.Contributors = app.ContributorSpecs{}
+	}
+
+	if schema.Registries == nil {
+		schema.Registries = app.RegistryRefSpecs{}
+	}
+
+	if schema.Libraries == nil {
+		schema.Libraries = app.LibraryRefSpecs{}
+	}
+
 	return &schema, nil
 }
 
@@ -315,7 +334,7 @@ func (m *manager) createUserDirTree() error {
 	return nil
 }
 
-func (m *manager) createAppDirTree(name string, gh registry.Manager) error {
+func (m *manager) createAppDirTree(name string, appYAMLData, baseLibData []byte, gh registry.Manager) error {
 	exists, err := afero.DirExists(m.appFS, string(m.rootPath))
 	if err != nil {
 		return fmt.Errorf("Could not check existance of directory '%s':\n%v", m.rootPath, err)
@@ -340,11 +359,6 @@ func (m *manager) createAppDirTree(name string, gh registry.Manager) error {
 		}
 	}
 
-	appYAML, err := genAppYAMLContent(name)
-	if err != nil {
-		return err
-	}
-
 	filePaths := []struct {
 		path    AbsPath
 		content []byte
@@ -359,7 +373,11 @@ func (m *manager) createAppDirTree(name string, gh registry.Manager) error {
 		},
 		{
 			m.appYAMLPath,
-			appYAML,
+			appYAMLData,
+		},
+		{
+			m.baseLibsonnetPath,
+			baseLibData,
 		},
 	}
 
@@ -400,12 +418,29 @@ func genComponentParamsContent() []byte {
 `)
 }
 
-func genAppYAMLContent(name string) ([]byte, error) {
+func generateRegistryYAMLData(incubatorReg registry.Manager) ([]byte, error) {
+	regSpec, err := incubatorReg.FetchRegistrySpec()
+	if err != nil {
+		return nil, err
+	}
+
+	return regSpec.Marshal()
+}
+
+func generateAppYAMLData(name string, refs ...*app.RegistryRefSpec) ([]byte, error) {
 	content := app.Spec{
 		APIVersion: app.DefaultAPIVersion,
 		Kind:       app.Kind,
 		Name:       name,
 		Version:    app.DefaultVersion,
+		Registries: app.RegistryRefSpecs{},
+	}
+
+	for _, ref := range refs {
+		err := content.AddRegistryRef(ref)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	return content.Marshal()

metadata/parts/schema.go

+// Copyright 2017 The kubecfg authors
+//
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+
+package parts
+
+import "encoding/json"
+
+const (
+	DefaultApiVersion = "0.1"
+	DefaultKind       = "ksonnet.io/parts"
+)
+
+type Spec struct {
+	APIVersion string `json:"apiVersion"`
+	Kind       string `json:"kind"`
+
+	Prototypes   PrototypeRefSpecs `json:"prototypes"`
+	Name         string            `json:"name"`
+	Version      string            `json:"version"`
+	Description  string            `json:"description"`
+	Author       string            `json:"author"`
+	Contributors ContributorSpecs  `json:"contributors"`
+	Repository   RepositorySpec    `json:"repository"`
+	Bugs         *BugSpec          `json:"bugs"`
+	Keywords     []string          `json:"keywords"`
+	QuickStart   *QuickStartSpec   `json:"quickStart"`
+	License      string            `json:"license"`
+}
+
+func (s *Spec) Marshal() ([]byte, error) {
+	return json.MarshalIndent(s, "", "  ")
+}
+
+type ContributorSpec struct {
+	Name  string `json:"name"`
+	Email string `json:"email"`
+}
+
+type ContributorSpecs []*ContributorSpec
+
+type RepositorySpec struct {
+	Type string `json:"type"`
+	URL  string `json:"url"`
+}
+
+type BugSpec struct {
+	URL string `json:"url"`
+}
+
+type QuickStartSpec struct {
+	Prototype     string            `json:"prototype"`
+	ComponentName string            `json:"componentName"`
+	Flags         map[string]string `json:"flags"`
+	Comment       string            `json:"comment"`
+}
+
+type Specs []*Spec
+
+type PrototypeRefSpecs map[string]string

metadata/registry.go

@@ -5,32 +5,43 @@ import (
 	"fmt"
 
 	"github.com/ksonnet/ksonnet/metadata/app"
+	"github.com/ksonnet/ksonnet/metadata/parts"
 	"github.com/ksonnet/ksonnet/metadata/registry"
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/afero"
 )
 
 // AddRegistry adds a registry with `name`, `protocol`, and `uri` to
 // the current ksonnet application.
-func (m *manager) AddRegistry(name, protocol, uri string) (*registry.Spec, error) {
-	app, err := m.AppSpec()
+func (m *manager) AddRegistry(name, protocol, uri, version string) (*registry.Spec, error) {
+	appSpec, err := m.AppSpec()
 	if err != nil {
 		return nil, err
 	}
 
-	// Retrieve or create registry specification.
-	registryRef, err := app.AddRegistryRef(name, protocol, uri)
+	// Add registry reference to app spec.
+	registryManager, err := makeGitHubRegistryManager(&app.RegistryRefSpec{
+		Name:     name,
+		Protocol: protocol,
+		URI:      uri,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	err = appSpec.AddRegistryRef(registryManager.RegistryRefSpec)
 	if err != nil {
 		return nil, err
 	}
 
 	// Retrieve the contents of registry.
-	registrySpec, err := m.getOrCacheRegistry(registryRef)
+	registrySpec, err := m.getOrCacheRegistry(registryManager)
 	if err != nil {
 		return nil, err
 	}
 
 	// Write registry specification back out to app specification.
-	specBytes, err := app.Marshal()
+	specBytes, err := appSpec.Marshal()
 	if err != nil {
 		return nil, err
 	}
@@ -43,47 +54,181 @@ func (m *manager) AddRegistry(name, protocol, uri string) (*registry.Spec, error
 	return registrySpec, nil
 }
 
+func (m *manager) GetRegistry(name string) (*registry.Spec, string, error) {
+	registryManager, protocol, err := m.getRegistryManager(name)
+	if err != nil {
+		return nil, "", err
+	}
+
+	regSpec, exists, err := m.registrySpecFromFile(m.registryPath(registryManager))
+	if !exists {
+		return nil, "", fmt.Errorf("Registry '%s' does not exist", name)
+	} else if err != nil {
+		return nil, "", err
+	}
+
+	return regSpec, protocol, nil
+}
+
+func (m *manager) CacheDependency(registryName, libID, libName, libVersion string) (*parts.Spec, error) {
+	// Retrieve application specification.
+	appSpec, err := m.AppSpec()
+	if err != nil {
+		return nil, err
+	}
+
+	if _, ok := appSpec.Libraries[libName]; ok {
+		return nil, fmt.Errorf("Library '%s' already exists", libName)
+	}
+
+	// Retrieve registry manager for this specific registry.
+	regRefSpec, exists := appSpec.GetRegistryRef(registryName)
+	if !exists {
+		return nil, fmt.Errorf("Registry '%s' does not exist", registryName)
+	}
+
+	registryManager, _, err := m.getRegistryManagerFor(regRefSpec)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get all directories and files first, then write to disk. This
+	// protects us from failing with a half-cached dependency because of
+	// a network failure.
+	directories := []AbsPath{}
+	files := map[AbsPath][]byte{}
+	parts, libRef, err := registryManager.ResolveLibrary(
+		libID,
+		libName,
+		libVersion,
+		func(relPath string, contents []byte) error {
+			files[appendToAbsPath(m.vendorPath, relPath)] = contents
+			return nil
+		},
+		func(relPath string) error {
+			directories = append(directories, appendToAbsPath(m.vendorPath, relPath))
+			return nil
+		})
+	if err != nil {
+		return nil, err
+	}
+
+	// Add library to app specification, but wait to write it out until
+	// the end, in case one of the network calls fails.
+	appSpec.Libraries[libName] = libRef
+	appSpecData, err := appSpec.Marshal()
+	if err != nil {
+		return nil, err
+	}
+
+	log.Infof("Retrieved %d files", len(files))
+
+	for _, dir := range directories {
+		if err := m.appFS.MkdirAll(string(dir), defaultFolderPermissions); err != nil {
+			return nil, err
+		}
+	}
+
+	for path, content := range files {
+		if err := afero.WriteFile(m.appFS, string(path), content, defaultFilePermissions); err != nil {
+			return nil, err
+		}
+	}
+
+	err = afero.WriteFile(m.appFS, string(m.appYAMLPath), appSpecData, defaultFilePermissions)
+	if err != nil {
+		return nil, err
+	}
+
+	return parts, nil
+}
+
 func (m *manager) registryDir(regManager registry.Manager) AbsPath {
-	return appendToAbsPath(m.registriesPath, regManager.VersionsDir())
+	return appendToAbsPath(m.registriesPath, regManager.RegistrySpecDir())
 }
 
 func (m *manager) registryPath(regManager registry.Manager) AbsPath {
-	return appendToAbsPath(m.registriesPath, regManager.SpecPath())
+	return appendToAbsPath(m.registriesPath, regManager.RegistrySpecFilePath())
 }
 
-func (m *manager) getOrCacheRegistry(registryRefSpec *app.RegistryRefSpec) (*registry.Spec, error) {
+func (m *manager) getRegistryManager(registryName string) (registry.Manager, string, error) {
+	appSpec, err := m.AppSpec()
+	if err != nil {
+		return nil, "", err
+	}
+
+	regRefSpec, exists := appSpec.GetRegistryRef(registryName)
+	if !exists {
+		return nil, "", fmt.Errorf("Registry '%s' does not exist", registryName)
+	}
+
+	return m.getRegistryManagerFor(regRefSpec)
+}
+
+func (m *manager) getRegistryManagerFor(registryRefSpec *app.RegistryRefSpec) (registry.Manager, string, error) {
+	var err error
+	var manager registry.Manager
+	var protocol string
+
 	switch registryRefSpec.Protocol {
 	case "github":
-		break
+		manager, err = makeGitHubRegistryManager(registryRefSpec)
+		protocol = "github"
 	default:
-		return nil, fmt.Errorf("Invalid protocol '%s'", registryRefSpec.Protocol)
+		return nil, "", fmt.Errorf("Invalid protocol '%s'", registryRefSpec.Protocol)
 	}
 
-	// Check local disk cache.
-	gh, err := makeGitHubRegistryManager(registryRefSpec)
 	if err != nil {
-		return nil, err
+		return nil, "", err
+	}
+
+	return manager, protocol, nil
+}
+
+func (m *manager) registrySpecFromFile(path AbsPath) (*registry.Spec, bool, error) {
+	registrySpecFile := string(path)
+	exists, err := afero.Exists(m.appFS, registrySpecFile)
+	if err != nil {
+		return nil, false, err
+	}
+
+	isDir, err := afero.IsDir(m.appFS, registrySpecFile)
+	if err != nil {
+		return nil, false, err
 	}
-	registrySpecFile := string(m.registryPath(gh))
 
-	exists, _ := afero.Exists(m.appFS, registrySpecFile)
-	isDir, _ := afero.IsDir(m.appFS, registrySpecFile)
+	// NOTE: case where directory of the same name exists should be
+	// fine, most filesystems allow you to have a directory and file of
+	// the same name.
 	if exists && !isDir {
 		registrySpecBytes, err := afero.ReadFile(m.appFS, registrySpecFile)
 		if err != nil {
-			return nil, err
+			return nil, false, err
 		}
 
 		registrySpec := registry.Spec{}
 		err = json.Unmarshal(registrySpecBytes, &registrySpec)
 		if err != nil {
-			return nil, err
+			return nil, false, err
 		}
-		return &registrySpec, nil
+		return &registrySpec, true, nil
+	}
+
+	return nil, false, nil
+}
+
+func (m *manager) getOrCacheRegistry(gh registry.Manager) (*registry.Spec, error) {
+	// Check local disk cache.
+	registrySpecFile := m.registryPath(gh)
+	registrySpec, exists, err := m.registrySpecFromFile(registrySpecFile)
+	if !exists {
+		return nil, fmt.Errorf("Registry '%s' does not exist", gh.MakeRegistryRefSpec().Name)
+	} else if err != nil {
+		return nil, err
 	}
 
 	// If failed, use the protocol to try to retrieve app specification.
-	registrySpec, err := gh.FindSpec()
+	registrySpec, err = gh.FetchRegistrySpec()
 	if err != nil {
 		return nil, err
 	}
@@ -96,13 +241,13 @@ func (m *manager) getOrCacheRegistry(registryRefSpec *app.RegistryRefSpec) (*reg
 	// NOTE: We call mkdir after getting the registry spec, since a
 	// network call might fail and leave this half-initialized empty
 	// directory.
-	registrySpecDir := appendToAbsPath(m.registriesPath, gh.VersionsDir())
+	registrySpecDir := appendToAbsPath(m.registriesPath, gh.RegistrySpecDir())
 	err = m.appFS.MkdirAll(string(registrySpecDir), defaultFolderPermissions)
 	if err != nil {
 		return nil, err
 	}
 
-	err = afero.WriteFile(m.appFS, registrySpecFile, registrySpecBytes, defaultFilePermissions)
+	err = afero.WriteFile(m.appFS, string(registrySpecFile), registrySpecBytes, defaultFilePermissions)
 	if err != nil {
 		return nil, err
 	}

metadata/registry/interface.go

+package registry
+
+import (
+	"github.com/ksonnet/ksonnet/metadata/app"
+	"github.com/ksonnet/ksonnet/metadata/parts"
+)
+
+type ResolveFile func(relPath string, contents []byte) error
+type ResolveDirectory func(relPath string) error
+
+type Manager interface {
+	RegistrySpecDir() string
+	RegistrySpecFilePath() string
+	FetchRegistrySpec() (*Spec, error)
+	MakeRegistryRefSpec() *app.RegistryRefSpec
+	ResolveLibrary(libID, libAlias, version string, onFile ResolveFile, onDir ResolveDirectory) (*parts.Spec, *app.LibraryRefSpec, error)
+}

metadata/registry/manager.go

-package registry
-
-type Manager interface {
-	VersionsDir() string
-	SpecPath() string
-	FindSpec() (*Spec, error)
-}

metadata/registry/schema.go

@@ -15,7 +15,11 @@
 
 package registry
 
-import "encoding/json"
+import (
+	"encoding/json"
+
+	"github.com/ksonnet/ksonnet/metadata/app"
+)
 
 const (
 	DefaultApiVersion = "0.1"
@@ -23,10 +27,10 @@ const (
 )
 
 type Spec struct {
-	APIVersion string            `json:"apiVersion"`
-	Kind       string            `json:"kind"`
-	Prototypes PrototypeRefSpecs `json:"prototypes"`
-	Libraries  LibraryRefSpecs   `json:"libraries"`
+	APIVersion string              `json:"apiVersion"`
+	Kind       string              `json:"kind"`
+	GitVersion *app.GitVersionSpec `json:"gitVersion"`
+	Libraries  LibraryRefSpecs     `json:"libraries"`
 }
 
 func (s *Spec) Marshal() ([]byte, error) {
@@ -35,6 +39,9 @@ func (s *Spec) Marshal() ([]byte, error) {
 
 type Specs []*Spec
 
-type LibraryRefSpecs map[string]string
+type LibraryRef struct {
+	Version string `json:"version"`
+	Path    string `json:"path"`
+}
 
-type PrototypeRefSpecs map[string]string
+type LibraryRefSpecs map[string]*LibraryRef

metadata/registry_managers.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/google/go-github/github"
 	"github.com/ksonnet/ksonnet/metadata/app"
+	"github.com/ksonnet/ksonnet/metadata/parts"
 	"github.com/ksonnet/ksonnet/metadata/registry"
 )
 
@@ -29,10 +30,9 @@ const (
 type gitHubRegistryManager struct {
 	*app.RegistryRefSpec
 	registryDir          string
-	RefSpec              string `json:"refSpec"`
-	ResolvedSHA          string `json:"resolvedSHA"`
 	org                  string
 	repo                 string
+	registryRepoPath     string
 	registrySpecRepoPath string
 }
 
@@ -42,50 +42,52 @@ func makeGitHubRegistryManager(registryRef *app.RegistryRefSpec) (*gitHubRegistr
 	var err error
 
 	// Set registry path.
-	// NOTE: Resolve this to a specific commit.
 	gh.registryDir = gh.Name
 
-	rawURI, uriExists := gh.Spec[uriField]
-	uri, isString := rawURI.(string)
-	if !uriExists || !isString {
-		return nil, fmt.Errorf("GitHub app registry '%s' is missing a 'uri' in field 'spec'", gh.Name)
+	// Parse GitHub URI.
+	var refspec string
+	gh.org, gh.repo, refspec, gh.registryRepoPath, gh.registrySpecRepoPath, err = parseGitHubURI(gh.URI)
+	if err != nil {
+		return nil, err
 	}
 
-	gh.org, gh.repo, gh.RefSpec, gh.registrySpecRepoPath, err = parseGitHubURI(uri)
+	// Resolve the refspec to a commit SHA.
+	client := github.NewClient(nil)
+	ctx := context.Background()
+
+	sha, _, err := client.Repositories.GetCommitSHA1(ctx, gh.org, gh.repo, refspec, "")
 	if err != nil {
 		return nil, err
 	}
+	gh.GitVersion = &app.GitVersionSpec{
+		RefSpec:   refspec,
+		CommitSHA: sha,
+	}
 
 	return &gh, nil
 }
 
-func (gh *gitHubRegistryManager) VersionsDir() string {
+func (gh *gitHubRegistryManager) RegistrySpecDir() string {
 	return gh.registryDir
 }
 
-func (gh *gitHubRegistryManager) SpecPath() string {
-	if gh.ResolvedSHA != "" {
-		return path.Join(gh.registryDir, gh.ResolvedSHA+".yaml")
+func (gh *gitHubRegistryManager) RegistrySpecFilePath() string {
+	if gh.GitVersion.CommitSHA != "" {
+		return path.Join(gh.registryDir, gh.GitVersion.CommitSHA+".yaml")
 	}
-	return path.Join(gh.registryDir, gh.RefSpec+".yaml")
+	return path.Join(gh.registryDir, gh.GitVersion.RefSpec+".yaml")
 }
 
-func (gh *gitHubRegistryManager) FindSpec() (*registry.Spec, error) {
+func (gh *gitHubRegistryManager) FetchRegistrySpec() (*registry.Spec, error) {
 	// Fetch app spec at specific commit.
 	client := github.NewClient(nil)
 	ctx := context.Background()
 
-	sha, _, err := client.Repositories.GetCommitSHA1(ctx, gh.org, gh.repo, gh.RefSpec, "")
-	if err != nil {
-		return nil, err
-	}
-	gh.ResolvedSHA = sha
-
 	// Get contents.
-	getOpts := github.RepositoryContentGetOptions{Ref: gh.ResolvedSHA}
+	getOpts := github.RepositoryContentGetOptions{Ref: gh.GitVersion.CommitSHA}
 	file, _, _, err := client.Repositories.GetContents(ctx, gh.org, gh.repo, gh.registrySpecRepoPath, &getOpts)
 	if file == nil {
-		return nil, fmt.Errorf("Could not find valid registry at uri '%s/%s/%s' and refspec '%s' (resolves to sha '%s')", gh.org, gh.repo, gh.registrySpecRepoPath, gh.RefSpec, gh.ResolvedSHA)
+		return nil, fmt.Errorf("Could not find valid registry at uri '%s/%s/%s' and refspec '%s' (resolves to sha '%s')", gh.org, gh.repo, gh.registrySpecRepoPath, gh.GitVersion.RefSpec, gh.GitVersion.CommitSHA)
 	} else if err != nil {
 		return nil, err
 	}
@@ -102,14 +104,114 @@ func (gh *gitHubRegistryManager) FindSpec() (*registry.Spec, error) {
 		return nil, err
 	}
 
+	registrySpec.GitVersion = &app.GitVersionSpec{
+		RefSpec:   gh.GitVersion.RefSpec,
+		CommitSHA: gh.GitVersion.CommitSHA,
+	}
+
 	return &registrySpec, nil
 }
 
+func (gh *gitHubRegistryManager) MakeRegistryRefSpec() *app.RegistryRefSpec {
+	return gh.RegistryRefSpec
+}
+
+func (gh *gitHubRegistryManager) ResolveLibrary(libID, libAlias, libRefSpec string, onFile registry.ResolveFile, onDir registry.ResolveDirectory) (*parts.Spec, *app.LibraryRefSpec, error) {
+	client := github.NewClient(nil)
+
+	// Resolve `version` (a git refspec) to a specific SHA.
+	ctx := context.Background()
+	resolvedSHA, _, err := client.Repositories.GetCommitSHA1(ctx, gh.org, gh.repo, libRefSpec, "")
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Resolve directories and files.
+	path := strings.Join([]string{gh.registryRepoPath, libID}, "/")
+	err = gh.resolveDir(client, libID, path, resolvedSHA, onFile, onDir)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Resolve app spec.
+	appSpecPath := strings.Join([]string{path, partsYAMLFile}, "/")
+	ctx = context.Background()
+	getOpts := &github.RepositoryContentGetOptions{Ref: resolvedSHA}
+	file, directory, _, err := client.Repositories.GetContents(ctx, gh.org, gh.repo, appSpecPath, getOpts)
+	if err != nil {
+		return nil, nil, err
+	} else if directory != nil {
+		return nil, nil, fmt.Errorf("Can't download library specification; resource '%s' points at a file", gh.registrySpecRawURL())
+	}
+
+	partsSpecText, err := file.GetContent()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	parts := parts.Spec{}
+	json.Unmarshal([]byte(partsSpecText), &parts)
+
+	refSpec := app.LibraryRefSpec{
+		Name: libAlias,
+		GitVersion: &app.GitVersionSpec{
+			RefSpec:   libRefSpec,
+			CommitSHA: resolvedSHA,
+		},
+	}
+
+	return &parts, &refSpec, nil
+}
+
+func (gh *gitHubRegistryManager) resolveDir(client *github.Client, libID, path, version string, onFile registry.ResolveFile, onDir registry.ResolveDirectory) error {
+	ctx := context.Background()
+	getOpts := &github.RepositoryContentGetOptions{Ref: version}
+	file, directory, _, err := client.Repositories.GetContents(ctx, gh.org, gh.repo, path, getOpts)
+	if err != nil {
+		return err
+	} else if file != nil {
+		return fmt.Errorf("Lib ID '%s' resolves to a file in registry '%s'", libID, gh.Name)
+	}
+
+	for _, item := range directory {
+		switch item.GetType() {
+		case "file":
+			itemPath := item.GetPath()
+			file, directory, _, err := client.Repositories.GetContents(ctx, gh.org, gh.repo, itemPath, getOpts)
+			if err != nil {
+				return err
+			} else if directory != nil {
+				return fmt.Errorf("INTERNAL ERROR: GitHub API reported resource '%s' of type file, but returned type dir", itemPath)
+			}
+			contents, err := file.GetContent()
+			if err != nil {
+				return err
+			}
+			if err := onFile(itemPath, []byte(contents)); err != nil {
+				return err
+			}
+		case "dir":
+			itemPath := item.GetPath()
+			if err := onDir(itemPath); err != nil {
+				return err
+			}
+			if err := gh.resolveDir(client, libID, itemPath, version, onFile, onDir); err != nil {
+				return err
+			}
+		case "symlink":
+		case "submodule":
+			return fmt.Errorf("Invalid library '%s'; ksonnet doesn't support libraries with symlinks or submodules", libID)
+		}
+	}
+
+	return nil
+}
+
 func (gh *gitHubRegistryManager) registrySpecRawURL() string {
-	return strings.Join([]string{rawGitHubRoot, gh.org, gh.repo, gh.RefSpec, gh.registrySpecRepoPath}, "/")
+	return strings.Join([]string{rawGitHubRoot, gh.org, gh.repo, gh.GitVersion.RefSpec, gh.registrySpecRepoPath}, "/")
 }
 
-func parseGitHubURI(uri string) (org, repo, refSpec, regSpecRepoPath string, err error) {
+func parseGitHubURI(uri string) (org, repo, refSpec, regRepoPath, regSpecRepoPath string, err error) {
 	// Normalize URI.
 	uri = strings.TrimSpace(uri)
 	if strings.HasPrefix(uri, "http://github.com") || strings.HasPrefix(uri, "https://github.com") || strings.HasPrefix(uri, "http://www.github.com") || strings.HasPrefix(uri, "https://www.github.com") {
@@ -117,21 +219,21 @@ func parseGitHubURI(uri string) (org, repo, refSpec, regSpecRepoPath string, err
 	} else if strings.HasPrefix(uri, "github.com") || strings.HasPrefix(uri, "www.github.com") {
 		uri = "http://" + uri
 	} else {
-		return "", "", "", "", fmt.Errorf("Registries using protocol 'github' must provide URIs beginning with 'github.com' (optionally prefaced with 'http', 'https', 'www', and so on")
+		return "", "", "", "", "", fmt.Errorf("Registries using protocol 'github' must provide URIs beginning with 'github.com' (optionally prefaced with 'http', 'https', 'www', and so on")
 	}
 
 	parsed, err := url.Parse(uri)
 	if err != nil {
-		return "", "", "", "", err
+		return "", "", "", "", "", err
 	}
 
 	if len(parsed.Query()) != 0 {
-		return "", "", "", "", fmt.Errorf("No query strings allowed in registry URI:\n%s", uri)
+		return "", "", "", "", "", fmt.Errorf("No query strings allowed in registry URI:\n%s", uri)
 	}
 
 	components := strings.Split(parsed.Path, "/")
 	if len(components) < 3 {
-		return "", "", "", "", fmt.Errorf("GitHub URI must point at a respository:\n%s", uri)
+		return "", "", "", "", "", fmt.Errorf("GitHub URI must point at a respository:\n%s", uri)
 	}
 
 	// NOTE: The first component is always blank, because the path
@@ -158,6 +260,8 @@ func parseGitHubURI(uri string) (org, repo, refSpec, regSpecRepoPath string, err
 
 		// See note above about first component being blank.
 		if components[3] == "tree" {
+			regRepoPath = strings.Join(components[5:], "/")
+
 			// If we have a trailing '/' character, last component will be
 			// blank.
 			if components[len-1] == "" {
@@ -168,11 +272,12 @@ func parseGitHubURI(uri string) (org, repo, refSpec, regSpecRepoPath string, err
 			regSpecRepoPath = strings.Join(components[5:], "/")
 			return
 		} else if components[3] == "blob" && components[len-1] == registryYAMLFile {
+			regRepoPath = strings.Join(components[5:len-1], "/")
 			// Path to the `registry.yaml` (may or may not exist).
 			regSpecRepoPath = strings.Join(components[5:], "/")
 			return
 		} else {
-			return "", "", "", "", fmt.Errorf("Invalid GitHub URI: try navigating in GitHub to the URI of the folder containing the 'app.yaml', and using that URI instead. Generally, this URI should be of the form 'github.com/{organization}/{repository}/tree/{branch}/[path-to-directory]'")
+			return "", "", "", "", "", fmt.Errorf("Invalid GitHub URI: try navigating in GitHub to the URI of the folder containing the 'app.yaml', and using that URI instead. Generally, this URI should be of the form 'github.com/{organization}/{repository}/tree/{branch}/[path-to-directory]'")
 		}
 	} else {
 		refSpec = defaultGitHubBranch
@@ -185,38 +290,8 @@ func parseGitHubURI(uri string) (org, repo, refSpec, regSpecRepoPath string, err
 			components = append(components, defaultGitHubBranch, registryYAMLFile)
 		}
 
+		regRepoPath = ""
 		regSpecRepoPath = registryYAMLFile
 		return
 	}
 }
-
-//
-// Mock registry manager.
-//
-
-type mockRegistryManager struct {
-	registryDir string
-}
-
-func newMockRegistryManager(name string) *mockRegistryManager {
-	return &mockRegistryManager{
-		registryDir: name,
-	}
-}
-
-func (m *mockRegistryManager) VersionsDir() string {
-	return m.registryDir
-}
-
-func (m *mockRegistryManager) SpecPath() string {
-	return path.Join(m.registryDir, "master.yaml")
-}
-
-func (m *mockRegistryManager) FindSpec() (*registry.Spec, error) {
-	registrySpec := registry.Spec{
-		APIVersion: registry.DefaultApiVersion,
-		Kind:       registry.DefaultKind,
-	}
-
-	return &registrySpec, nil
-}

metadata/registry_test.go

 package metadata
 
 import (
-	"testing"
+	"path"
 
 	"github.com/ksonnet/ksonnet/metadata/app"
+	"github.com/ksonnet/ksonnet/metadata/parts"
+	"github.com/ksonnet/ksonnet/metadata/registry"
 )
 
-type ghRegistryGetSuccess struct {
-	target string
-	source string
+type mockRegistryManager struct {
+	*app.RegistryRefSpec
+	registryDir string
 }
 
-func (r *ghRegistryGetSuccess) Test(t *testing.T) {
-	gh, err := makeGitHubRegistryManager(&app.RegistryRefSpec{
-		Protocol: "github",
-		Spec: map[string]interface{}{
-			uriField:     r.source,
-			refSpecField: "master",
+func newMockRegistryManager(name string) *mockRegistryManager {
+	return &mockRegistryManager{
+		registryDir: name,
+		RegistryRefSpec: &app.RegistryRefSpec{
+			Name: name,
 		},
-		Name: "incubator",
-	})
-	if err != nil {
-		t.Error(err)
-	}
-
-	rawURI := gh.registrySpecRawURL()
-	if rawURI != r.target {
-		t.Errorf("Expected URI '%s', got '%s'", r.target, rawURI)
 	}
 }
 
-func TestGetRegistryRefSuccess(t *testing.T) {
-	successes := []*ghRegistryGetSuccess{
-		&ghRegistryGetSuccess{
-			source: "http://github.com/ksonnet/parts",
-			target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
-		},
-		&ghRegistryGetSuccess{
-			source: "http://github.com/ksonnet/parts/",
-			target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
-		},
-		&ghRegistryGetSuccess{
-			source: "http://www.github.com/ksonnet/parts",
-			target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
-		},
-		&ghRegistryGetSuccess{
-			source: "https://www.github.com/ksonnet/parts",
-			target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
-		},
-		&ghRegistryGetSuccess{
-			source: "github.com/ksonnet/parts",
-			target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
-		},
-		&ghRegistryGetSuccess{
-			source: "www.github.com/ksonnet/parts",
-			target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
-		},
+func (m *mockRegistryManager) RegistrySpecDir() string {
+	return m.registryDir
+}
 
-		&ghRegistryGetSuccess{
-			source: "http://github.com/ksonnet/parts/tree/master/incubator",
-			target: "https://raw.githubusercontent.com/ksonnet/parts/master/incubator/registry.yaml",
-		},
-		&ghRegistryGetSuccess{
-			source: "http://github.com/ksonnet/parts/tree/master/incubator/",
-			target: "https://raw.githubusercontent.com/ksonnet/parts/master/incubator/registry.yaml",
-		},
-		&ghRegistryGetSuccess{
-			source: "http://www.github.com/ksonnet/parts/tree/master/incubator",
-			target: "https://raw.githubusercontent.com/ksonnet/parts/master/incubator/registry.yaml",
-		},
-		&ghRegistryGetSuccess{
-			source: "https://github.com/ksonnet/parts/tree/master/incubator",
-			target: "https://raw.githubusercontent.com/ksonnet/parts/master/incubator/registry.yaml",
-		},
-		&ghRegistryGetSuccess{
-			source: "github.com/ksonnet/parts/tree/master/incubator",
-			target: "https://raw.githubusercontent.com/ksonnet/parts/master/incubator/registry.yaml",
-		},
-		&ghRegistryGetSuccess{
-			source: "www.github.com/ksonnet/parts/tree/master/incubator",
-			target: "https://raw.githubusercontent.com/ksonnet/parts/master/incubator/registry.yaml",
-		},
+func (m *mockRegistryManager) RegistrySpecFilePath() string {
+	return path.Join(m.registryDir, "master.yaml")
+}
 
-		&ghRegistryGetSuccess{
-			source: "http://github.com/ksonnet/parts/blob/master/registry.yaml",
-			target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
-		},
-		&ghRegistryGetSuccess{
-			source: "http://www.github.com/ksonnet/parts/blob/master/registry.yaml",
-			target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
-		},
-		&ghRegistryGetSuccess{
-			source: "https://github.com/ksonnet/parts/blob/master/registry.yaml",
-			target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
-		},
-		&ghRegistryGetSuccess{
-			source: "github.com/ksonnet/parts/blob/master/registry.yaml",
-			target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
-		},
-		&ghRegistryGetSuccess{
-			source: "www.github.com/ksonnet/parts/blob/master/registry.yaml",
-			target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
-		},
+func (m *mockRegistryManager) FetchRegistrySpec() (*registry.Spec, error) {
+	registrySpec := registry.Spec{
+		APIVersion: registry.DefaultApiVersion,
+		Kind:       registry.DefaultKind,
 	}
 
-	for _, success := range successes {
-		success.Test(t)
-	}
+	return &registrySpec, nil
 }
 
-//
-// TODO: Add failure tests, like:
-//
-// &ghRegistryGetSuccess{
-// 	source: "http://github.com/ksonnet/parts/tree/master/incubator?foo=bar",
-// },
-//
-
-func TestCacheGitHubRegistry(t *testing.T) {
-	// registrySpec, err := cacheGitHubRegistry("")
-	// if err != nil {
-	// 	t.Error(err)
-	// }
+func (m *mockRegistryManager) MakeRegistryRefSpec() *app.RegistryRefSpec {
+	return m.RegistryRefSpec
+}
 
-	// panic(registrySpec)
+func (m *mockRegistryManager) ResolveLibrary(libID, libAlias, version string, onFile registry.ResolveFile, onDir registry.ResolveDirectory) (*parts.Spec, *app.LibraryRefSpec, error) {
+	return nil, nil, nil
 }