Create helm registry for ksonnet

Creates a helm registry that can treat helm charts as ksonnet parts.

[design doc](https://docs.google.com/document/d/1FcK8pDqj_9EFjNVr9e_t9WemouQpS9ix-r0AxNDB_nM/edit?usp=sharing)

Signed-off-by: bryanl bryanliles@gmail.com

pkg/app/schema.go

@@ -238,7 +238,7 @@ type EnvironmentSpecs map[string]*EnvironmentSpec
 type EnvironmentSpec struct {
 	// Name is the user defined name of an environment
 	Name string `json:"-"`
-	// KubernetesVersion is the kubernetes version the targetted cluster is
+	// KubernetesVersion is the kubernetes version the targeted cluster is
 	// running on.
 	KubernetesVersion string `json:"k8sVersion"`
 	// Path is the relative project path containing metadata for this

pkg/cluster/apply.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/ksonnet/ksonnet/pkg/app"
 	"github.com/ksonnet/ksonnet/pkg/client"
+	"github.com/ksonnet/ksonnet/pkg/metadata"
 	"github.com/ksonnet/ksonnet/utils"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
@@ -40,9 +41,6 @@ import (
 )
 
 const (
-	annotationManaged  = "ksonnet.io/managed"
-	labelDeployManager = "app.kubernetes.io/deploy-manager"
-
 	appKsonnet = "ksonnet"
 )
 
@@ -189,7 +187,7 @@ func (a *Apply) handleObject(co clientOpts, obj *unstructured.Unstructured) (str
 	}
 
 	if a.GcTag != "" {
-		SetMetaDataAnnotation(mergedObject, AnnotationGcTag, a.GcTag)
+		SetMetaDataAnnotation(mergedObject, metadata.AnnotationGcTag, a.GcTag)
 	}
 
 	desc := fmt.Sprintf("%s %s", a.objectInfo.ResourceName(co.discovery, mergedObject), utils.FqName(obj))
@@ -248,8 +246,8 @@ func tagManaged(obj *unstructured.Unstructured) error {
 		return err
 	}
 
-	SetMetaDataLabel(obj, labelDeployManager, appKsonnet)
-	SetMetaDataAnnotation(obj, annotationManaged, string(mmEncoded))
+	SetMetaDataLabel(obj, metadata.LabelDeployManager, appKsonnet)
+	SetMetaDataAnnotation(obj, metadata.AnnotationManaged, string(mmEncoded))
 
 	return nil
 }

pkg/cluster/cluster.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/ksonnet/ksonnet/pkg/app"
 	"github.com/ksonnet/ksonnet/pkg/client"
+	"github.com/ksonnet/ksonnet/pkg/metadata"
 	"github.com/ksonnet/ksonnet/pkg/pipeline"
 	"github.com/ksonnet/ksonnet/utils"
 	log "github.com/sirupsen/logrus"
@@ -33,23 +34,6 @@ import (
 	"k8s.io/client-go/dynamic"
 )
 
-const (
-	// AnnotationGcTag annotation that triggers
-	// garbage collection. Objects with value equal to
-	// command-line flag that are *not* in config will be deleted.
-	AnnotationGcTag = "kubecfg.ksonnet.io/garbage-collect-tag"
-
-	// AnnotationGcStrategy controls gc logic.  Current values:
-	// `auto` (default if absent) - do garbage collection
-	// `ignore` - never garbage collect this object
-	AnnotationGcStrategy = "kubecfg.ksonnet.io/garbage-collect-strategy"
-
-	// GcStrategyAuto is the default automatic gc logic
-	GcStrategyAuto = "auto"
-	// GcStrategyIgnore means this object should be ignored by garbage collection
-	GcStrategyIgnore = "ignore"
-)
-
 type genClientOptsFn func(a app.App, c *client.Config, envName string) (clientOpts, error)
 
 type clientFactoryFn func(dynamic.ClientPool, discovery.DiscoveryInterface, runtime.Object, string) (dynamic.ResourceInterface, error)
@@ -177,11 +161,11 @@ func eligibleForGc(obj metav1.Object, gcTag string) bool {
 
 	a := obj.GetAnnotations()
 
-	strategy, ok := a[AnnotationGcStrategy]
+	strategy, ok := a[metadata.AnnotationGcStrategy]
 	if !ok {
-		strategy = GcStrategyAuto
+		strategy = metadata.GcStrategyAuto
 	}
 
-	return a[AnnotationGcTag] == gcTag &&
-		strategy == GcStrategyAuto
+	return a[metadata.AnnotationGcTag] == gcTag &&
+		strategy == metadata.GcStrategyAuto
 }

pkg/cluster/object.go

@@ -18,6 +18,7 @@ package cluster
 import (
 	"encoding/json"
 
+	clustermetadata "github.com/ksonnet/ksonnet/pkg/metadata"
 	"github.com/pkg/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -114,7 +115,7 @@ func RebuildObject(m map[string]interface{}) (map[string]interface{}, error) {
 	if !ok {
 		return nil, errors.New("metadata annotations not found")
 	}
-	descriptor, ok := annotations[annotationManaged].(string)
+	descriptor, ok := annotations[clustermetadata.AnnotationManaged].(string)
 	if !ok {
 		return m, nil
 	}

pkg/helm/client.go

+// Copyright 2018 The ksonnet authors
+//
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+
+package helm
+
+import (
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"path"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/blang/semver"
+	"github.com/ghodss/yaml"
+	"github.com/pkg/errors"
+)
+
+var (
+	validHelmSchemes = map[string]bool{
+		"http":  true,
+		"https": true,
+	}
+)
+
+// RepositoryClient is a client for retrieving Helm repository contents. The structure
+// is loosely documented at https://github.com/kubernetes/helm/blob/master/docs/chart_repository.md.
+type RepositoryClient interface {
+	// Repository returns the contents of a Helm repository.
+	Repository() (*Repository, error)
+	// Chart returns a Chart with a given name and version. If the version is blank, it returns
+	// the latest version.
+	Chart(name, version string) (*RepositoryChart, error)
+	// Fetch fetches URIs for a chart.
+	Fetch(uri string) (io.ReadCloser, error)
+}
+
+// Repository is metadata describing the contents of a Helm repository.
+type Repository struct {
+	Charts map[string][]RepositoryChart `json:"entries,omitempty"`
+}
+
+// Latest returns the latest version of charts in a Helm repository.
+func (hr *Repository) Latest() []RepositoryChart {
+	var out []RepositoryChart
+
+	for _, chart := range hr.Charts {
+		if len(chart) > 0 {
+			sort.Sort(RepositoryCharts(chart))
+			out = append(out, chart[0])
+		}
+	}
+
+	return out
+}
+
+// RepositoryChart is metadata describing a Helm Chart in a repository.
+type RepositoryChart struct {
+	Description string   `json:"description,omitempty"`
+	Name        string   `json:"name,omitempty"`
+	URLs        []string `json:"urls,omitempty"`
+	Version     string   `json:"version,omitempty"`
+}
+
+// RepositoryCharts is a slice of RepositoryChart.
+type RepositoryCharts []RepositoryChart
+
+func (rc RepositoryCharts) Len() int {
+	return len(rc)
+}
+
+func (rc RepositoryCharts) Swap(i, j int) { rc[i], rc[j] = rc[j], rc[i] }
+
+func (rc RepositoryCharts) Less(i, j int) bool {
+	v1, err := semver.Make(rc[i].Version)
+	if err != nil {
+		return false
+	}
+	v2, err := semver.Make(rc[j].Version)
+	if err != nil {
+		return false
+	}
+
+	return v1.Compare(v2) == 1
+}
+
+type Getter interface {
+	Get(string) (*http.Response, error)
+}
+
+type httpGetter struct {
+	client *http.Client
+}
+
+func newHTTPGetter() *httpGetter {
+	c := &http.Client{
+		Timeout: 30 * time.Second,
+	}
+
+	return &httpGetter{
+		client: c,
+	}
+}
+
+func (g *httpGetter) Get(s string) (*http.Response, error) {
+	req, err := http.NewRequest(http.MethodGet, s, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return g.client.Do(req)
+}
+
+// HTTPClient is a HTTP Helm repository client.
+type HTTPClient struct {
+	url    *url.URL
+	getter Getter
+}
+
+var _ RepositoryClient = (*HTTPClient)(nil)
+
+// NewHTTPClient creates an instance of HTTPClient
+func NewHTTPClient(urlStr string, hg Getter) (*HTTPClient, error) {
+	normalized, err := normalizeHelmURI(urlStr)
+	if err != nil {
+		return nil, errors.Wrap(err, "normalizing Helm repository URL")
+	}
+
+	u, err := url.Parse(normalized)
+	if err != nil {
+		return nil, err
+	}
+
+	if hg == nil {
+		hg = newHTTPGetter()
+	}
+
+	return &HTTPClient{
+		url:    u,
+		getter: hg,
+	}, nil
+}
+
+// Repository returns the Helm repository's content.
+func (hrc *HTTPClient) Repository() (*Repository, error) {
+	r, err := hrc.Fetch(hrc.url.String())
+	if r != nil {
+		defer r.Close()
+	}
+
+	if err != nil {
+		return nil, errors.Wrap(err, "retrieving repository index")
+	}
+
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, errors.Wrap(err, "reading repository index.yaml body")
+	}
+
+	var hre Repository
+
+	if err := yaml.Unmarshal(b, &hre); err != nil {
+		return nil, errors.Wrap(err, "unmarshalling repository index.yaml")
+	}
+
+	return &hre, nil
+}
+
+// Chart returns a chart from the repository. If version is blank, it returns the latest.
+func (hrc *HTTPClient) Chart(name, version string) (*RepositoryChart, error) {
+	repo, err := hrc.Repository()
+	if err != nil {
+		return nil, errors.Wrap(err, "retrieving repository")
+	}
+
+	if version == "" {
+		for _, chart := range repo.Latest() {
+			if name == chart.Name {
+				return &chart, nil
+			}
+		}
+
+		return nil, errors.Errorf("chart %q was not found", name)
+	}
+
+	charts, ok := repo.Charts[name]
+	if !ok {
+		return nil, errors.Errorf("chart %q was not found", name)
+	}
+
+	for _, chart := range charts {
+		if version == chart.Version {
+			return &chart, nil
+		}
+	}
+
+	return nil, errors.Errorf("chart %q with version %q was not found", name, version)
+}
+
+// Fetch fetches URLs from a repository. If uri is a path, it will use the client URL as the base.
+func (hrc *HTTPClient) Fetch(uri string) (io.ReadCloser, error) {
+	u, err := url.Parse(uri)
+	if err != nil {
+		return nil, err
+	}
+
+	if u.Host == "" {
+		*u = *hrc.url
+		u.Path = uri
+	}
+
+	resp, err := hrc.getter.Get(u.String())
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.Errorf("unexpected HTTP status code %q when retrieving %q",
+			resp.Status, u.String())
+	}
+
+	return resp.Body, nil
+}
+
+// normalizeHelmURI normalizes a Helm repository URI by returning the
+// full URL to repository's index.yaml file.
+func normalizeHelmURI(s string) (string, error) {
+	u, err := url.Parse(s)
+	if err != nil {
+		return "", errors.Wrapf(err, "parsing Helm URI %q", s)
+	}
+
+	if _, ok := validHelmSchemes[u.Scheme]; !ok {
+		return "", errors.Errorf("%q is an invalid scheme for Helm repository", u.Scheme)
+	}
+
+	if strings.HasSuffix(u.Path, "index.yaml") {
+		return s, nil
+	}
+
+	u.Path = path.Join(u.Path, "index.yaml")
+	return u.String(), nil
+}

pkg/helm/client_test.go

+// Copyright 2018 The ksonnet authors
+//
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+
+package helm
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_normalizeHelmURI(t *testing.T) {
+	cases := []struct {
+		name     string
+		uri      string
+		expected string
+		isErr    bool
+	}{
+		{
+			name:     "host with index.yaml",
+			uri:      "http://host/index.yaml",
+			expected: "http://host/index.yaml",
+		},
+		{
+			name:     "host with index.yaml in nested path",
+			uri:      "http://host/nested/index.yaml",
+			expected: "http://host/nested/index.yaml",
+		},
+		{
+			name:     "host with no index.yaml",
+			uri:      "http://host",
+			expected: "http://host/index.yaml",
+		},
+		{
+			name:     "host with path and no index.yaml",
+			uri:      "http://host/nested",
+			expected: "http://host/nested/index.yaml",
+		},
+		{
+			name:  "invalid URL",
+			uri:   "ht tp://host",
+			isErr: true,
+		},
+		{
+			name:  "not a URL",
+			uri:   "invalid",
+			isErr: true,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := normalizeHelmURI(tc.uri)
+			if tc.isErr {
+				require.Error(t, err)
+				return
+			}
+
+			require.NoError(t, err)
+
+			require.Equal(t, tc.expected, got)
+		})
+	}
+}
+
+func TestRepository_Latest(t *testing.T) {
+	r := Repository{
+		Charts: map[string][]RepositoryChart{
+			"app-a": []RepositoryChart{
+				{
+					Name:    "app-a",
+					Version: "0.2.0",
+				},
+				{
+					Name:    "app-a",
+					Version: "0.3.0",
+				},
+			},
+		},
+	}
+
+	got := r.Latest()
+
+	expected := []RepositoryChart{{Name: "app-a", Version: "0.3.0"}}
+
+	require.Equal(t, expected, got)
+}
+
+func Test_httpGetter_Get(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, "response")
+	}))
+
+	defer ts.Close()
+
+	g := newHTTPGetter()
+
+	r, err := g.Get(ts.URL)
+	require.NoError(t, err)
+
+	b, err := ioutil.ReadAll(r.Body)
+	require.NoError(t, err)
+	defer r.Body.Close()
+
+	assert.Equal(t, http.StatusOK, r.StatusCode)
+	assert.Equal(t, "response", string(b))
+}
+
+func genCharts() RepositoryCharts {
+	return RepositoryCharts{
+		{
+			Name:    "app-a",
+			Version: "0.2.0",
+		},
+		{
+			Name:    "app-a",
+			Version: "0.3.0",
+		},
+	}
+}
+
+func TestRepositoryCharts_Sort_Len(t *testing.T) {
+	assert.Equal(t, 2, genCharts().Len())
+}
+
+func TestRepositoryCharts_Sort_Swap(t *testing.T) {
+	charts := genCharts()
+
+	charts.Swap(0, 1)
+	expected := RepositoryCharts{
+		{
+			Name:    "app-a",
+			Version: "0.3.0",
+		},
+		{
+			Name:    "app-a",
+			Version: "0.2.0",
+		},
+	}
+
+	assert.Equal(t, expected, charts)
+}
+
+func TestRepositoryCharts_Sort_Less(t *testing.T) {
+	charts := RepositoryCharts{
+		{
+			Name:    "app-a",
+			Version: "0.3.0",
+		},
+		{
+			Name:    "app-a",
+			Version: "0.2.0",
+		},
+		{
+			Name: "app-a",
+		},
+	}
+
+	cases := []struct {
+		name     string
+		i        int
+		j        int
+		expected bool
+	}{
+		{
+			name:     "i version is greater than j",
+			i:        0,
+			j:        1,
+			expected: true,
+		},
+		{
+			name:     "i version is less than j",
+			i:        1,
+			j:        0,
+			expected: false,
+		},
+		{
+			name:     "i version is equal to j",
+			i:        1,
+			j:        1,
+			expected: false,
+		},
+		{
+			name:     "i version is invalid",
+			i:        2,
+			j:        1,
+			expected: false,
+		},
+		{
+			name:     "j version is invalid",
+			i:        1,
+			j:        2,
+			expected: false,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := charts.Less(tc.i, tc.j)
+			require.Equal(t, tc.expected, got)
+		})
+	}
+}
+
+func Test_repositoryClient_Repository(t *testing.T) {
+	cases := []struct {
+		name    string
+		httpGet func(*testing.T) Getter
+		isErr   bool
+	}{
+		{
+			name: "entries were found and are valid",
+			httpGet: func(t *testing.T) Getter {
+				f, err := os.Open(filepath.ToSlash("testdata/index.yaml"))
+				require.NoError(t, err)
+
+				r := &http.Response{
+					Body:       f,
+					Status:     http.StatusText(http.StatusOK),
+					StatusCode: http.StatusOK,
+				}
+
+				return &fakeGetter{getResponse: r}
+			},
+		},
+		{
+			name: "get failed",
+			httpGet: func(t *testing.T) Getter {
+				return &fakeGetter{getErr: errors.New("failed")}
+			},
+			isErr: true,
+		},