From b87f6d15b140a7befad8a008038cab709315d12a Mon Sep 17 00:00:00 2001
From: Alex Clemmer <clemmer.alexander@gmail.com>
Date: Tue, 31 Oct 2017 13:44:20 -0700
Subject: [PATCH] Introduce `registry.Manager` abstraction + impl for GitHub
The vast majority of ksonnet apps will know about at least one registry,
the official ksonnet `incubator` registry. In general, managing
registries can involve fairly complex logic (e.g., resolving a reference
to a registry to a remote registry specification; finding all
libraries exposed by a registry; and so on).
This commit will introduce the `registry.Manager` abstraction, as well
as an implementation for registries hosted as GitHub repositories.
---
metadata/interface.go | 31 +++--
metadata/manager.go | 24 +++-
metadata/registry.go | 111 +++++++++++++++++
metadata/registry/manager.go | 7 ++
metadata/registry/schema.go | 6 +
metadata/registry_managers.go | 222 ++++++++++++++++++++++++++++++++++
metadata/registry_test.go | 127 +++++++++++++++++++
7 files changed, 515 insertions(+), 13 deletions(-)
create mode 100644 metadata/registry.go
create mode 100644 metadata/registry/manager.go
create mode 100644 metadata/registry_managers.go
create mode 100644 metadata/registry_test.go
diff --git a/metadata/interface.go b/metadata/interface.go
index c2bd1adc..3e65f0f8 100644
--- a/metadata/interface.go
+++ b/metadata/interface.go
@@ -20,7 +20,9 @@ import (
"regexp"
"strings"
+ "github.com/ksonnet/ksonnet/metadata/app"
param "github.com/ksonnet/ksonnet/metadata/params"
+ "github.com/ksonnet/ksonnet/metadata/registry"
"github.com/ksonnet/ksonnet/prototype"
"github.com/spf13/afero"
)
@@ -42,17 +44,16 @@ type AbsPaths []string
// libraries; and other non-core-application tasks.
type Manager interface {
Root() AbsPath
+ LibPaths(envName string) (libPath, envLibPath, envComponentPath, envParamsPath AbsPath)
+
+ // Components API.
ComponentPaths() (AbsPaths, error)
CreateComponent(name string, text string, params param.Params, templateType prototype.TemplateType) error
- LibPaths(envName string) (libPath, envLibPath, envComponentPath, envParamsPath AbsPath)
+
+ // Params API.
SetComponentParams(component string, params param.Params) error
GetComponentParams(name string) (param.Params, error)
GetAllComponentParams() (map[string]param.Params, error)
- CreateEnvironment(name, uri, namespace string, spec ClusterSpec) error
- DeleteEnvironment(name string) error
- GetEnvironments() ([]*Environment, error)
- GetEnvironment(name string) (*Environment, error)
- SetEnvironment(name string, desired *Environment) error
// GetEnvironmentParams will take the name of an environment and return a
// mapping of parameters of the form:
// componentName => {param key => param val}
@@ -60,12 +61,18 @@ type Manager interface {
GetEnvironmentParams(name string) (map[string]param.Params, error)
SetEnvironmentParams(env, component string, params param.Params) error
- //
- // TODO: Fill in methods as we need them.
- //
- // GetPrototype(id string) Protoype
- // SearchPrototypes(query string) []Protoype
- // VendorLibrary(uri, version string) error
+ // Environment API.
+ CreateEnvironment(name, uri, namespace string, spec ClusterSpec) error
+ DeleteEnvironment(name string) error
+ GetEnvironments() ([]*Environment, error)
+ GetEnvironment(name string) (*Environment, error)
+ SetEnvironment(name string, desired *Environment) error
+
+ // Spec API.
+ AppSpec() (*app.Spec, error)
+
+ // Registry API.
+ AddRegistry(name, protocol, uri string) (*registry.Spec, error)
}
// Find will recursively search the current directory and its parents for a
diff --git a/metadata/manager.go b/metadata/manager.go
index ae5da1ab..7356085a 100644
--- a/metadata/manager.go
+++ b/metadata/manager.go
@@ -38,6 +38,7 @@ func appendToAbsPath(originalPath AbsPath, toAppend ...string) AbsPath {
const (
ksonnetDir = ".ksonnet"
+ registriesDir = ksonnetDir + "/registries"
libDir = "lib"
componentsDir = "components"
environmentsDir = "environments"
@@ -46,6 +47,7 @@ const (
componentParamsFile = "params.libsonnet"
baseLibsonnetFile = "base.libsonnet"
appYAMLFile = "app.yaml"
+ registryYAMLFile = "registry.yaml"
// ComponentsExtCodeKey is the ExtCode key for component imports
ComponentsExtCodeKey = "__ksonnet/components"
@@ -63,6 +65,7 @@ type manager struct {
// Application paths.
rootPath AbsPath
ksonnetPath AbsPath
+ registriesPath AbsPath
libPath AbsPath
componentsPath AbsPath
environmentsPath AbsPath
@@ -151,6 +154,7 @@ func newManager(rootPath AbsPath, appFS afero.Fs) (*manager, error) {
// Application paths.
rootPath: rootPath,
ksonnetPath: appendToAbsPath(rootPath, ksonnetDir),
+ registriesPath: appendToAbsPath(rootPath, registriesDir),
libPath: appendToAbsPath(rootPath, libDir),
componentsPath: appendToAbsPath(rootPath, componentsDir),
environmentsPath: appendToAbsPath(rootPath, environmentsDir),
@@ -260,6 +264,23 @@ func (m *manager) SetComponentParams(component string, params param.Params) erro
return afero.WriteFile(m.appFS, string(m.componentParamsPath), []byte(jsonnet), defaultFilePermissions)
}
+// AppSpec will return the specification for a ksonnet application
+// (typically stored in `app.yaml`)
+func (m *manager) AppSpec() (*app.Spec, error) {
+ bytes, err := afero.ReadFile(m.appFS, string(m.appYAMLPath))
+ if err != nil {
+ return nil, err
+ }
+
+ schema := app.Spec{}
+ err = json.Unmarshal(bytes, &schema)
+ if err != nil {
+ return nil, err
+ }
+
+ return &schema, nil
+}
+
func (m *manager) createUserDirTree() error {
dirPaths := []AbsPath{
m.userKsonnetRootPath,
@@ -286,6 +307,7 @@ func (m *manager) createAppDirTree(name string) error {
dirPaths := []AbsPath{
m.rootPath,
m.ksonnetPath,
+ m.registriesPath,
m.libPath,
m.componentsPath,
m.environmentsPath,
@@ -366,7 +388,7 @@ func genAppYAMLContent(name string) ([]byte, error) {
Version: app.DefaultVersion,
}
- return json.MarshalIndent(&content, "", " ")
+ return content.Marshal()
}
func genBaseLibsonnetContent() []byte {
diff --git a/metadata/registry.go b/metadata/registry.go
new file mode 100644
index 00000000..f60512cc
--- /dev/null
+++ b/metadata/registry.go
@@ -0,0 +1,111 @@
+package metadata
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "github.com/ksonnet/ksonnet/metadata/app"
+ "github.com/ksonnet/ksonnet/metadata/registry"
+ "github.com/spf13/afero"
+)
+
+// AddRegistry adds a registry with `name`, `protocol`, and `uri` to
+// the current ksonnet application.
+func (m *manager) AddRegistry(name, protocol, uri string) (*registry.Spec, error) {
+ app, err := m.AppSpec()
+ if err != nil {
+ return nil, err
+ }
+
+ // Retrieve or create registry specification.
+ registryRef, err := app.AddRegistryRef(name, protocol, uri)
+ if err != nil {
+ return nil, err
+ }
+
+ // Retrieve the contents of registry.
+ registrySpec, err := m.getOrCacheRegistry(registryRef)
+ if err != nil {
+ return nil, err
+ }
+
+ // Write registry specification back out to app specification.
+ specBytes, err := app.Marshal()
+ if err != nil {
+ return nil, err
+ }
+
+ err = afero.WriteFile(m.appFS, string(m.appYAMLPath), specBytes, defaultFilePermissions)
+ if err != nil {
+ return nil, err
+ }
+
+ return registrySpec, nil
+}
+
+func (m *manager) registryDir(regManager registry.Manager) AbsPath {
+ return appendToAbsPath(m.registriesPath, regManager.VersionsDir())
+}
+
+func (m *manager) registryPath(regManager registry.Manager) AbsPath {
+ return appendToAbsPath(m.registriesPath, regManager.SpecPath())
+}
+
+func (m *manager) getOrCacheRegistry(registryRefSpec *app.RegistryRefSpec) (*registry.Spec, error) {
+ switch registryRefSpec.Protocol {
+ case "github":
+ break
+ default:
+ return nil, fmt.Errorf("Invalid protocol '%s'", registryRefSpec.Protocol)
+ }
+
+ // Check local disk cache.
+ gh, err := makeGitHubRegistryManager(registryRefSpec)
+ if err != nil {
+ return nil, err
+ }
+ registrySpecFile := string(m.registryPath(gh))
+
+ exists, _ := afero.Exists(m.appFS, registrySpecFile)
+ isDir, _ := afero.IsDir(m.appFS, registrySpecFile)
+ if exists && !isDir {
+ registrySpecBytes, err := afero.ReadFile(m.appFS, registrySpecFile)
+ if err != nil {
+ return nil, err
+ }
+
+ registrySpec := registry.Spec{}
+ err = json.Unmarshal(registrySpecBytes, ®istrySpec)
+ if err != nil {
+ return nil, err
+ }
+ return ®istrySpec, nil
+ }
+
+ // If failed, use the protocol to try to retrieve app specification.
+ registrySpec, err := gh.FindSpec()
+ if err != nil {
+ return nil, err
+ }
+
+ registrySpecBytes, err := registrySpec.Marshal()
+ if err != nil {
+ return nil, err
+ }
+
+ // NOTE: We call mkdir after getting the registry spec, since a
+ // network call might fail and leave this half-initialized empty
+ // directory.
+ registrySpecDir := appendToAbsPath(m.registriesPath, gh.VersionsDir())
+ err = m.appFS.MkdirAll(string(registrySpecDir), defaultFolderPermissions)
+ if err != nil {
+ return nil, err
+ }
+
+ err = afero.WriteFile(m.appFS, registrySpecFile, registrySpecBytes, defaultFilePermissions)
+ if err != nil {
+ return nil, err
+ }
+
+ return registrySpec, nil
+}
diff --git a/metadata/registry/manager.go b/metadata/registry/manager.go
new file mode 100644
index 00000000..16232cd3
--- /dev/null
+++ b/metadata/registry/manager.go
@@ -0,0 +1,7 @@
+package registry
+
+type Manager interface {
+ VersionsDir() string
+ SpecPath() string
+ FindSpec() (*Spec, error)
+}
diff --git a/metadata/registry/schema.go b/metadata/registry/schema.go
index e63d203a..1c31fee4 100644
--- a/metadata/registry/schema.go
+++ b/metadata/registry/schema.go
@@ -15,6 +15,8 @@
package registry
+import "encoding/json"
+
const (
DefaultApiVersion = "0.1"
DefaultKind = "ksonnet.io/registry"
@@ -27,6 +29,10 @@ type Spec struct {
Libraries LibraryRefSpecs `json:"libraries"`
}
+func (s *Spec) Marshal() ([]byte, error) {
+ return json.MarshalIndent(s, "", " ")
+}
+
type Specs []*Spec
type LibraryRefSpecs map[string]string
diff --git a/metadata/registry_managers.go b/metadata/registry_managers.go
new file mode 100644
index 00000000..a1714514
--- /dev/null
+++ b/metadata/registry_managers.go
@@ -0,0 +1,222 @@
+package metadata
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/url"
+ "path"
+ "strings"
+
+ "github.com/google/go-github/github"
+ "github.com/ksonnet/ksonnet/metadata/app"
+ "github.com/ksonnet/ksonnet/metadata/registry"
+)
+
+const (
+ rawGitHubRoot = "https://raw.githubusercontent.com"
+ defaultGitHubBranch = "master"
+
+ uriField = "uri"
+ refSpecField = "refSpec"
+ resolvedSHAField = "resolvedSHA"
+)
+
+//
+// GitHub registry manager.
+//
+
+type gitHubRegistryManager struct {
+ *app.RegistryRefSpec
+ registryDir string
+ RefSpec string `json:"refSpec"`
+ ResolvedSHA string `json:"resolvedSHA"`
+ org string
+ repo string
+ registrySpecRepoPath string
+}
+
+func makeGitHubRegistryManager(registryRef *app.RegistryRefSpec) (*gitHubRegistryManager, error) {
+ gh := gitHubRegistryManager{RegistryRefSpec: registryRef}
+
+ var err error
+
+ // Set registry path.
+ // NOTE: Resolve this to a specific commit.
+ gh.registryDir = gh.Name
+
+ rawURI, uriExists := gh.Spec[uriField]
+ uri, isString := rawURI.(string)
+ if !uriExists || !isString {
+ return nil, fmt.Errorf("GitHub app registry '%s' is missing a 'uri' in field 'spec'", gh.Name)
+ }
+
+ gh.org, gh.repo, gh.RefSpec, gh.registrySpecRepoPath, err = parseGitHubURI(uri)
+ if err != nil {
+ return nil, err
+ }
+
+ return &gh, nil
+}
+
+func (gh *gitHubRegistryManager) VersionsDir() string {
+ return gh.registryDir
+}
+
+func (gh *gitHubRegistryManager) SpecPath() string {
+ if gh.ResolvedSHA != "" {
+ return path.Join(gh.registryDir, gh.ResolvedSHA+".yaml")
+ }
+ return path.Join(gh.registryDir, gh.RefSpec+".yaml")
+}
+
+func (gh *gitHubRegistryManager) FindSpec() (*registry.Spec, error) {
+ // Fetch app spec at specific commit.
+ client := github.NewClient(nil)
+ ctx := context.Background()
+
+ sha, _, err := client.Repositories.GetCommitSHA1(ctx, gh.org, gh.repo, gh.RefSpec, "")
+ if err != nil {
+ return nil, err
+ }
+ gh.ResolvedSHA = sha
+
+ // Get contents.
+ getOpts := github.RepositoryContentGetOptions{Ref: gh.ResolvedSHA}
+ file, _, _, err := client.Repositories.GetContents(ctx, gh.org, gh.repo, gh.registrySpecRepoPath, &getOpts)
+ if file == nil {
+ return nil, fmt.Errorf("Could not find valid registry at uri '%s/%s/%s' and refspec '%s' (resolves to sha '%s')", gh.org, gh.repo, gh.registrySpecRepoPath, gh.RefSpec, gh.ResolvedSHA)
+ } else if err != nil {
+ return nil, err
+ }
+
+ registrySpecText, err := file.GetContent()
+ if err != nil {
+ return nil, err
+ }
+
+ // Deserialize, return.
+ registrySpec := registry.Spec{}
+ err = json.Unmarshal([]byte(registrySpecText), ®istrySpec)
+ if err != nil {
+ return nil, err
+ }
+
+ return ®istrySpec, nil
+}
+
+func (gh *gitHubRegistryManager) registrySpecRawURL() string {
+ return strings.Join([]string{rawGitHubRoot, gh.org, gh.repo, gh.RefSpec, gh.registrySpecRepoPath}, "/")
+}
+
+func parseGitHubURI(uri string) (org, repo, refSpec, regSpecRepoPath string, err error) {
+ // Normalize URI.
+ uri = strings.TrimSpace(uri)
+ if strings.HasPrefix(uri, "http://github.com") || strings.HasPrefix(uri, "https://github.com") || strings.HasPrefix(uri, "http://www.github.com") || strings.HasPrefix(uri, "https://www.github.com") {
+ // Do nothing.
+ } else if strings.HasPrefix(uri, "github.com") || strings.HasPrefix(uri, "www.github.com") {
+ uri = "http://" + uri
+ } else {
+ return "", "", "", "", fmt.Errorf("Registries using protocol 'github' must provide URIs beginning with 'github.com' (optionally prefaced with 'http', 'https', 'www', and so on")
+ }
+
+ parsed, err := url.Parse(uri)
+ if err != nil {
+ return "", "", "", "", err
+ }
+
+ if len(parsed.Query()) != 0 {
+ return "", "", "", "", fmt.Errorf("No query strings allowed in registry URI:\n%s", uri)
+ }
+
+ components := strings.Split(parsed.Path, "/")
+ if len(components) < 3 {
+ return "", "", "", "", fmt.Errorf("GitHub URI must point at a respository:\n%s", uri)
+ }
+
+ // NOTE: The first component is always blank, because the path
+ // begins like: '/whatever'.
+ org = components[1]
+ repo = components[2]
+
+ //
+ // Parse out `regSpecRepoPath`. There are a few cases:
+ // * URI points at a directory inside the respoitory, e.g.,
+ // 'http://github.com/ksonnet/parts/tree/master/incubator'
+ // * URI points at an 'app.yaml', e.g.,
+ // 'http://github.com/ksonnet/parts/blob/master/app.yaml'
+ // * URI points at a repository root, e.g.,
+ // 'http://github.com/ksonnet/parts'
+ //
+ if len := len(components); len > 4 {
+ refSpec = components[4]
+
+ //
+ // Case where we're pointing at either a directory inside a GitHub
+ // URL, or an 'app.yaml' inside a GitHub URL.
+ //
+
+ // See note above about first component being blank.
+ if components[3] == "tree" {
+ // If we have a trailing '/' character, last component will be
+ // blank.
+ if components[len-1] == "" {
+ components[len-1] = registryYAMLFile
+ } else {
+ components = append(components, registryYAMLFile)
+ }
+ regSpecRepoPath = strings.Join(components[5:], "/")
+ return
+ } else if components[3] == "blob" && components[len-1] == registryYAMLFile {
+ // Path to the `registry.yaml` (may or may not exist).
+ regSpecRepoPath = strings.Join(components[5:], "/")
+ return
+ } else {
+ return "", "", "", "", fmt.Errorf("Invalid GitHub URI: try navigating in GitHub to the URI of the folder containing the 'app.yaml', and using that URI instead. Generally, this URI should be of the form 'github.com/{organization}/{repository}/tree/{branch}/[path-to-directory]'")
+ }
+ } else {
+ refSpec = defaultGitHubBranch
+
+ // Else, URI should point at repository root.
+ if components[len-1] == "" {
+ components[len-1] = defaultGitHubBranch
+ components = append(components, registryYAMLFile)
+ } else {
+ components = append(components, defaultGitHubBranch, registryYAMLFile)
+ }
+
+ regSpecRepoPath = registryYAMLFile
+ return
+ }
+}
+
+//
+// Mock registry manager.
+//
+
+type mockRegistryManager struct {
+ registryDir string
+}
+
+func newMockRegistryManager(name string) *mockRegistryManager {
+ return &mockRegistryManager{
+ registryDir: name,
+ }
+}
+
+func (m *mockRegistryManager) VersionsDir() string {
+ return m.registryDir
+}
+
+func (m *mockRegistryManager) SpecPath() string {
+ return path.Join(m.registryDir, "master.yaml")
+}
+
+func (m *mockRegistryManager) FindSpec() (*registry.Spec, error) {
+ registrySpec := registry.Spec{
+ APIVersion: registry.DefaultApiVersion,
+ Kind: registry.DefaultKind,
+ }
+
+ return ®istrySpec, nil
+}
diff --git a/metadata/registry_test.go b/metadata/registry_test.go
new file mode 100644
index 00000000..840ddf5f
--- /dev/null
+++ b/metadata/registry_test.go
@@ -0,0 +1,127 @@
+package metadata
+
+import (
+ "testing"
+
+ "github.com/ksonnet/ksonnet/metadata/app"
+)
+
+type ghRegistryGetSuccess struct {
+ target string
+ source string
+}
+
+func (r *ghRegistryGetSuccess) Test(t *testing.T) {
+ gh, err := makeGitHubRegistryManager(&app.RegistryRefSpec{
+ Protocol: "github",
+ Spec: map[string]interface{}{
+ uriField: r.source,
+ refSpecField: "master",
+ },
+ Name: "incubator",
+ })
+ if err != nil {
+ t.Error(err)
+ }
+
+ rawURI := gh.registrySpecRawURL()
+ if rawURI != r.target {
+ t.Errorf("Expected URI '%s', got '%s'", r.target, rawURI)
+ }
+}
+
+func TestGetRegistryRefSuccess(t *testing.T) {
+ successes := []*ghRegistryGetSuccess{
+ &ghRegistryGetSuccess{
+ source: "http://github.com/ksonnet/parts",
+ target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
+ },
+ &ghRegistryGetSuccess{
+ source: "http://github.com/ksonnet/parts/",
+ target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
+ },
+ &ghRegistryGetSuccess{
+ source: "http://www.github.com/ksonnet/parts",
+ target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
+ },
+ &ghRegistryGetSuccess{
+ source: "https://www.github.com/ksonnet/parts",
+ target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
+ },
+ &ghRegistryGetSuccess{
+ source: "github.com/ksonnet/parts",
+ target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
+ },
+ &ghRegistryGetSuccess{
+ source: "www.github.com/ksonnet/parts",
+ target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
+ },
+
+ &ghRegistryGetSuccess{
+ source: "http://github.com/ksonnet/parts/tree/master/incubator",
+ target: "https://raw.githubusercontent.com/ksonnet/parts/master/incubator/registry.yaml",
+ },
+ &ghRegistryGetSuccess{
+ source: "http://github.com/ksonnet/parts/tree/master/incubator/",
+ target: "https://raw.githubusercontent.com/ksonnet/parts/master/incubator/registry.yaml",
+ },
+ &ghRegistryGetSuccess{
+ source: "http://www.github.com/ksonnet/parts/tree/master/incubator",
+ target: "https://raw.githubusercontent.com/ksonnet/parts/master/incubator/registry.yaml",
+ },
+ &ghRegistryGetSuccess{
+ source: "https://github.com/ksonnet/parts/tree/master/incubator",
+ target: "https://raw.githubusercontent.com/ksonnet/parts/master/incubator/registry.yaml",
+ },
+ &ghRegistryGetSuccess{
+ source: "github.com/ksonnet/parts/tree/master/incubator",
+ target: "https://raw.githubusercontent.com/ksonnet/parts/master/incubator/registry.yaml",
+ },
+ &ghRegistryGetSuccess{
+ source: "www.github.com/ksonnet/parts/tree/master/incubator",
+ target: "https://raw.githubusercontent.com/ksonnet/parts/master/incubator/registry.yaml",
+ },
+
+ &ghRegistryGetSuccess{
+ source: "http://github.com/ksonnet/parts/blob/master/registry.yaml",
+ target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
+ },
+ &ghRegistryGetSuccess{
+ source: "http://www.github.com/ksonnet/parts/blob/master/registry.yaml",
+ target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
+ },
+ &ghRegistryGetSuccess{
+ source: "https://github.com/ksonnet/parts/blob/master/registry.yaml",
+ target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
+ },
+ &ghRegistryGetSuccess{
+ source: "github.com/ksonnet/parts/blob/master/registry.yaml",
+ target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
+ },
+ &ghRegistryGetSuccess{
+ source: "www.github.com/ksonnet/parts/blob/master/registry.yaml",
+ target: "https://raw.githubusercontent.com/ksonnet/parts/master/registry.yaml",
+ },
+ }
+
+ for _, success := range successes {
+ success.Test(t)
+ }
+}
+
+//
+// TODO: Add failure tests, like:
+//
+// &ghRegistryGetSuccess{
+// source: "http://github.com/ksonnet/parts/tree/master/incubator?foo=bar",
+// },
+//
+
+func TestCacheGitHubRegistry(t *testing.T) {
+ // registrySpec, err := cacheGitHubRegistry("")
+ // if err != nil {
+ // t.Error(err)
+ // }
+
+ // panic(registrySpec)
+}
--
GitLab