Unverified Commit f6da2d55 authored by bryanl's avatar bryanl
Browse files

Create helm registry for ksonnet

Creates a helm registry that can treat helm charts as ksonnet parts.

[design doc](https://docs.google.com/document/d/1FcK8pDqj_9EFjNVr9e_t9WemouQpS9ix-r0AxNDB_nM/edit?usp=sharing)

Signed-off-by: bryanl bryanliles@gmail.com
parent 1270243e
......@@ -238,7 +238,7 @@ type EnvironmentSpecs map[string]*EnvironmentSpec
type EnvironmentSpec struct {
// Name is the user defined name of an environment
Name string `json:"-"`
// KubernetesVersion is the kubernetes version the targetted cluster is
// KubernetesVersion is the kubernetes version the targeted cluster is
// running on.
KubernetesVersion string `json:"k8sVersion"`
// Path is the relative project path containing metadata for this
......
......@@ -25,6 +25,7 @@ import (
"github.com/ksonnet/ksonnet/pkg/app"
"github.com/ksonnet/ksonnet/pkg/client"
"github.com/ksonnet/ksonnet/pkg/metadata"
"github.com/ksonnet/ksonnet/utils"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
......@@ -40,9 +41,6 @@ import (
)
const (
annotationManaged = "ksonnet.io/managed"
labelDeployManager = "app.kubernetes.io/deploy-manager"
appKsonnet = "ksonnet"
)
......@@ -189,7 +187,7 @@ func (a *Apply) handleObject(co clientOpts, obj *unstructured.Unstructured) (str
}
if a.GcTag != "" {
SetMetaDataAnnotation(mergedObject, AnnotationGcTag, a.GcTag)
SetMetaDataAnnotation(mergedObject, metadata.AnnotationGcTag, a.GcTag)
}
desc := fmt.Sprintf("%s %s", a.objectInfo.ResourceName(co.discovery, mergedObject), utils.FqName(obj))
......@@ -248,8 +246,8 @@ func tagManaged(obj *unstructured.Unstructured) error {
return err
}
SetMetaDataLabel(obj, labelDeployManager, appKsonnet)
SetMetaDataAnnotation(obj, annotationManaged, string(mmEncoded))
SetMetaDataLabel(obj, metadata.LabelDeployManager, appKsonnet)
SetMetaDataAnnotation(obj, metadata.AnnotationManaged, string(mmEncoded))
return nil
}
......
......@@ -20,6 +20,7 @@ import (
"github.com/ksonnet/ksonnet/pkg/app"
"github.com/ksonnet/ksonnet/pkg/client"
"github.com/ksonnet/ksonnet/pkg/metadata"
"github.com/ksonnet/ksonnet/pkg/pipeline"
"github.com/ksonnet/ksonnet/utils"
log "github.com/sirupsen/logrus"
......@@ -33,23 +34,6 @@ import (
"k8s.io/client-go/dynamic"
)
const (
// AnnotationGcTag annotation that triggers
// garbage collection. Objects with value equal to
// command-line flag that are *not* in config will be deleted.
AnnotationGcTag = "kubecfg.ksonnet.io/garbage-collect-tag"
// AnnotationGcStrategy controls gc logic. Current values:
// `auto` (default if absent) - do garbage collection
// `ignore` - never garbage collect this object
AnnotationGcStrategy = "kubecfg.ksonnet.io/garbage-collect-strategy"
// GcStrategyAuto is the default automatic gc logic
GcStrategyAuto = "auto"
// GcStrategyIgnore means this object should be ignored by garbage collection
GcStrategyIgnore = "ignore"
)
type genClientOptsFn func(a app.App, c *client.Config, envName string) (clientOpts, error)
type clientFactoryFn func(dynamic.ClientPool, discovery.DiscoveryInterface, runtime.Object, string) (dynamic.ResourceInterface, error)
......@@ -177,11 +161,11 @@ func eligibleForGc(obj metav1.Object, gcTag string) bool {
a := obj.GetAnnotations()
strategy, ok := a[AnnotationGcStrategy]
strategy, ok := a[metadata.AnnotationGcStrategy]
if !ok {
strategy = GcStrategyAuto
strategy = metadata.GcStrategyAuto
}
return a[AnnotationGcTag] == gcTag &&
strategy == GcStrategyAuto
return a[metadata.AnnotationGcTag] == gcTag &&
strategy == metadata.GcStrategyAuto
}
......@@ -18,6 +18,7 @@ package cluster
import (
"encoding/json"
clustermetadata "github.com/ksonnet/ksonnet/pkg/metadata"
"github.com/pkg/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
......@@ -114,7 +115,7 @@ func RebuildObject(m map[string]interface{}) (map[string]interface{}, error) {
if !ok {
return nil, errors.New("metadata annotations not found")
}
descriptor, ok := annotations[annotationManaged].(string)
descriptor, ok := annotations[clustermetadata.AnnotationManaged].(string)
if !ok {
return m, nil
}
......
// Copyright 2018 The ksonnet authors
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package helm
import (
"io"
"io/ioutil"
"net/http"
"net/url"
"path"
"sort"
"strings"
"time"
"github.com/blang/semver"
"github.com/ghodss/yaml"
"github.com/pkg/errors"
)
var (
validHelmSchemes = map[string]bool{
"http": true,
"https": true,
}
)
// RepositoryClient is a client for retrieving Helm repository contents. The structure
// is loosely documented at https://github.com/kubernetes/helm/blob/master/docs/chart_repository.md.
type RepositoryClient interface {
// Repository returns the contents of a Helm repository.
Repository() (*Repository, error)
// Chart returns a Chart with a given name and version. If the version is blank, it returns
// the latest version.
Chart(name, version string) (*RepositoryChart, error)
// Fetch fetches URIs for a chart.
Fetch(uri string) (io.ReadCloser, error)
}
// Repository is metadata describing the contents of a Helm repository.
type Repository struct {
Charts map[string][]RepositoryChart `json:"entries,omitempty"`
}
// Latest returns the latest version of charts in a Helm repository.
func (hr *Repository) Latest() []RepositoryChart {
var out []RepositoryChart
for _, chart := range hr.Charts {
if len(chart) > 0 {
sort.Sort(RepositoryCharts(chart))
out = append(out, chart[0])
}
}
return out
}
// RepositoryChart is metadata describing a Helm Chart in a repository.
type RepositoryChart struct {
Description string `json:"description,omitempty"`
Name string `json:"name,omitempty"`
URLs []string `json:"urls,omitempty"`
Version string `json:"version,omitempty"`
}
// RepositoryCharts is a slice of RepositoryChart.
type RepositoryCharts []RepositoryChart
func (rc RepositoryCharts) Len() int {
return len(rc)
}
func (rc RepositoryCharts) Swap(i, j int) { rc[i], rc[j] = rc[j], rc[i] }
func (rc RepositoryCharts) Less(i, j int) bool {
v1, err := semver.Make(rc[i].Version)
if err != nil {
return false
}
v2, err := semver.Make(rc[j].Version)
if err != nil {
return false
}
return v1.Compare(v2) == 1
}
type Getter interface {
Get(string) (*http.Response, error)
}
type httpGetter struct {
client *http.Client
}
func newHTTPGetter() *httpGetter {
c := &http.Client{
Timeout: 30 * time.Second,
}
return &httpGetter{
client: c,
}
}
func (g *httpGetter) Get(s string) (*http.Response, error) {
req, err := http.NewRequest(http.MethodGet, s, nil)
if err != nil {
return nil, err
}
return g.client.Do(req)
}
// HTTPClient is a HTTP Helm repository client.
type HTTPClient struct {
url *url.URL
getter Getter
}
var _ RepositoryClient = (*HTTPClient)(nil)
// NewHTTPClient creates an instance of HTTPClient
func NewHTTPClient(urlStr string, hg Getter) (*HTTPClient, error) {
normalized, err := normalizeHelmURI(urlStr)
if err != nil {
return nil, errors.Wrap(err, "normalizing Helm repository URL")
}
u, err := url.Parse(normalized)
if err != nil {
return nil, err
}
if hg == nil {
hg = newHTTPGetter()
}
return &HTTPClient{
url: u,
getter: hg,
}, nil
}
// Repository returns the Helm repository's content.
func (hrc *HTTPClient) Repository() (*Repository, error) {
r, err := hrc.Fetch(hrc.url.String())
if r != nil {
defer r.Close()
}
if err != nil {
return nil, errors.Wrap(err, "retrieving repository index")
}
b, err := ioutil.ReadAll(r)
if err != nil {
return nil, errors.Wrap(err, "reading repository index.yaml body")
}
var hre Repository
if err := yaml.Unmarshal(b, &hre); err != nil {
return nil, errors.Wrap(err, "unmarshalling repository index.yaml")
}
return &hre, nil
}
// Chart returns a chart from the repository. If version is blank, it returns the latest.
func (hrc *HTTPClient) Chart(name, version string) (*RepositoryChart, error) {
repo, err := hrc.Repository()
if err != nil {
return nil, errors.Wrap(err, "retrieving repository")
}
if version == "" {
for _, chart := range repo.Latest() {
if name == chart.Name {
return &chart, nil
}
}
return nil, errors.Errorf("chart %q was not found", name)
}
charts, ok := repo.Charts[name]
if !ok {
return nil, errors.Errorf("chart %q was not found", name)
}
for _, chart := range charts {
if version == chart.Version {
return &chart, nil
}
}
return nil, errors.Errorf("chart %q with version %q was not found", name, version)
}
// Fetch fetches URLs from a repository. If uri is a path, it will use the client URL as the base.
func (hrc *HTTPClient) Fetch(uri string) (io.ReadCloser, error) {
u, err := url.Parse(uri)
if err != nil {
return nil, err
}
if u.Host == "" {
*u = *hrc.url
u.Path = uri
}
resp, err := hrc.getter.Get(u.String())
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, errors.Errorf("unexpected HTTP status code %q when retrieving %q",
resp.Status, u.String())
}
return resp.Body, nil
}
// normalizeHelmURI normalizes a Helm repository URI by returning the
// full URL to repository's index.yaml file.
func normalizeHelmURI(s string) (string, error) {
u, err := url.Parse(s)
if err != nil {
return "", errors.Wrapf(err, "parsing Helm URI %q", s)
}
if _, ok := validHelmSchemes[u.Scheme]; !ok {
return "", errors.Errorf("%q is an invalid scheme for Helm repository", u.Scheme)
}
if strings.HasSuffix(u.Path, "index.yaml") {
return s, nil
}
u.Path = path.Join(u.Path, "index.yaml")
return u.String(), nil
}
// Copyright 2018 The ksonnet authors
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package helm
import (
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_normalizeHelmURI(t *testing.T) {
cases := []struct {
name string
uri string
expected string
isErr bool
}{
{
name: "host with index.yaml",
uri: "http://host/index.yaml",
expected: "http://host/index.yaml",
},
{
name: "host with index.yaml in nested path",
uri: "http://host/nested/index.yaml",
expected: "http://host/nested/index.yaml",
},
{
name: "host with no index.yaml",
uri: "http://host",
expected: "http://host/index.yaml",
},
{
name: "host with path and no index.yaml",
uri: "http://host/nested",
expected: "http://host/nested/index.yaml",
},
{
name: "invalid URL",
uri: "ht tp://host",
isErr: true,
},
{
name: "not a URL",
uri: "invalid",
isErr: true,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got, err := normalizeHelmURI(tc.uri)
if tc.isErr {
require.Error(t, err)
return
}
require.NoError(t, err)
require.Equal(t, tc.expected, got)
})
}
}
func TestRepository_Latest(t *testing.T) {
r := Repository{
Charts: map[string][]RepositoryChart{
"app-a": []RepositoryChart{
{
Name: "app-a",
Version: "0.2.0",
},
{
Name: "app-a",
Version: "0.3.0",
},
},
},
}
got := r.Latest()
expected := []RepositoryChart{{Name: "app-a", Version: "0.3.0"}}
require.Equal(t, expected, got)
}
func Test_httpGetter_Get(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, "response")
}))
defer ts.Close()
g := newHTTPGetter()
r, err := g.Get(ts.URL)
require.NoError(t, err)
b, err := ioutil.ReadAll(r.Body)
require.NoError(t, err)
defer r.Body.Close()
assert.Equal(t, http.StatusOK, r.StatusCode)
assert.Equal(t, "response", string(b))
}
func genCharts() RepositoryCharts {
return RepositoryCharts{
{
Name: "app-a",
Version: "0.2.0",
},
{
Name: "app-a",
Version: "0.3.0",
},
}
}
func TestRepositoryCharts_Sort_Len(t *testing.T) {
assert.Equal(t, 2, genCharts().Len())
}
func TestRepositoryCharts_Sort_Swap(t *testing.T) {
charts := genCharts()
charts.Swap(0, 1)
expected := RepositoryCharts{
{
Name: "app-a",
Version: "0.3.0",
},
{
Name: "app-a",
Version: "0.2.0",
},
}
assert.Equal(t, expected, charts)
}
func TestRepositoryCharts_Sort_Less(t *testing.T) {
charts := RepositoryCharts{
{
Name: "app-a",
Version: "0.3.0",
},
{
Name: "app-a",
Version: "0.2.0",
},
{
Name: "app-a",
},
}
cases := []struct {
name string
i int
j int
expected bool
}{
{
name: "i version is greater than j",
i: 0,
j: 1,
expected: true,
},
{
name: "i version is less than j",
i: 1,
j: 0,
expected: false,
},
{
name: "i version is equal to j",
i: 1,
j: 1,
expected: false,
},
{
name: "i version is invalid",
i: 2,
j: 1,
expected: false,
},
{
name: "j version is invalid",
i: 1,
j: 2,
expected: false,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := charts.Less(tc.i, tc.j)
require.Equal(t, tc.expected, got)
})
}
}
func Test_repositoryClient_Repository(t *testing.T) {
cases := []struct {
name string
httpGet func(*testing.T) Getter
isErr bool
}{
{
name: "entries were found and are valid",
httpGet: func(t *testing.T) Getter {
f, err := os.Open(filepath.ToSlash("testdata/index.yaml"))
require.NoError(t, err)
r := &http.Response{
Body: f,
Status: http.StatusText(http.StatusOK),
StatusCode: http.StatusOK,
}
return &fakeGetter{getResponse: r}
},
},
{
name: "get failed",
httpGet: func(t *testing.T) Getter {
return &fakeGetter{getErr: errors.New("failed")}
},
isErr: true,
},