Commit 6295812a authored by bryanl's avatar bryanl
Browse files

Introduce component namespaces



re: #292
Signed-off-by: default avatarbryanl <bryanliles@gmail.com>
parent 9bb797ae
......@@ -30,8 +30,7 @@ GUESTBOOK_FILE = examples/guestbook.jsonnet
DOC_GEN_FILE = ./docs/generate/update-generated-docs.sh
DOC_TEST_FILE = ./docs/generate/verify-generated-docs.sh
JSONNET_FILES = $(KCFG_TEST_FILE) $(GUESTBOOK_FILE)
# TODO: Simplify this once ./... ignores ./vendor
GO_PACKAGES = ./cmd/... ./utils/... ./pkg/... ./metadata/... ./prototype/...
GO_PACKAGES = ./...
# Default cluster from this config is used for integration tests
KUBECONFIG = $(HOME)/.kube/config
......@@ -60,7 +59,7 @@ integrationtest: ks
$(GINKGO) -tags 'integration' integration -- -fixtures $(INTEGRATION_TEST_FIXTURES) -kubeconfig $(KUBECONFIG) -ksonnet-bin $(abspath $<)
vet:
$(GO) vet $(GO_FLAGS) $(GO_PACKAGES)
$(GO) vet $(GO_PACKAGES)
fmt:
$(GOFMT) -s -w $(shell $(GO) list -f '{{.Dir}}' $(GO_PACKAGES))
......
......@@ -20,6 +20,8 @@ import (
"os"
"strings"
"github.com/ksonnet/ksonnet/component"
"github.com/spf13/pflag"
"github.com/ksonnet/ksonnet/metadata"
......@@ -417,7 +419,9 @@ var prototypeUseCmd = &cobra.Command{
return err
}
text, err := expandPrototype(proto, templateType, params, componentName)
_, prototypeName := component.ExtractNamespacedComponent(appFs, cwd, componentName)
text, err := expandPrototype(proto, templateType, params, prototypeName)
if err != nil {
return err
}
......
......@@ -38,6 +38,7 @@ import (
"k8s.io/client-go/dynamic"
"k8s.io/client-go/tools/clientcmd"
"github.com/ksonnet/ksonnet/component"
"github.com/ksonnet/ksonnet/metadata"
str "github.com/ksonnet/ksonnet/strings"
"github.com/ksonnet/ksonnet/template"
......@@ -433,40 +434,54 @@ func (te *cmdObjExpander) Expand() ([]*unstructured.Unstructured, error) {
}
_, vendorPath := manager.LibPaths()
libPath, mainPath, paramsPath, err := manager.EnvPaths(te.config.env)
libPath, mainPath, _, err := manager.EnvPaths(te.config.env)
if err != nil {
return nil, err
}
expander.FlagJpath = append([]string{string(vendorPath), string(libPath)}, expander.FlagJpath...)
componentPaths, err := manager.ComponentPaths()
namespacedComponentPaths, err := component.MakePathsByNamespace(te.config.fs, manager, te.config.cwd, te.config.env)
if err != nil {
return nil, errors.Wrap(err, "component paths")
}
baseObj, err := constructBaseObj(componentPaths, te.config.components)
if err != nil {
return nil, errors.Wrap(err, "construct base object")
}
//
// Set up ExtCodes to resolve runtime variables such as the environment namespace.
//
params := importParams(string(paramsPath))
envSpec, err := importEnv(manager, te.config.env)
if err != nil {
return nil, err
}
expander.ExtCodes = append([]string{baseObj, params, envSpec}, expander.ExtCodes...)
baseCodes := expander.ExtCodes
slUnstructured := make([]*unstructured.Unstructured, 0)
for ns, componentPaths := range namespacedComponentPaths {
paramsPath := ns.ParamsPath()
params := importParams(string(paramsPath))
baseObj, err := constructBaseObj(componentPaths, te.config.components)
if err != nil {
return nil, errors.Wrap(err, "construct base object")
}
//
// Expand the ksonnet app as rendered for environment `env`.
//
expander.ExtCodes = append([]string{baseObj, params, envSpec}, baseCodes...)
u, err := expander.Expand([]string{string(mainPath)})
if err != nil {
return nil, errors.Wrapf(err, "generate objects for namespace %s", ns.Path)
}
slUnstructured = append(slUnstructured, u...)
}
return slUnstructured, nil
return expander.Expand([]string{string(mainPath)})
}
// constructBaseObj constructs the base Jsonnet object that represents k-v
......
// Copyright 2017 The kubecfg authors
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package component
import (
"os"
"path/filepath"
"sort"
"strings"
"github.com/ksonnet/ksonnet/metadata/app"
"github.com/pkg/errors"
"github.com/spf13/afero"
)
const (
// componentsDir is the name of the directory which houses components.
componentsRoot = "components"
// paramsFile is the params file for a component namespace.
paramsFile = "params.libsonnet"
)
// Path returns returns the file system path for a component.
func Path(fs afero.Fs, root, name string) (string, error) {
ns, localName := ExtractNamespacedComponent(fs, root, name)
fis, err := afero.ReadDir(fs, ns.Dir())
if err != nil {
return "", err
}
var fileName string
files := make(map[string]bool)
for _, fi := range fis {
if fi.IsDir() {
continue
}
base := strings.TrimSuffix(fi.Name(), filepath.Ext(fi.Name()))
if _, ok := files[base]; ok {
return "", errors.Errorf("Found multiple component files with component name %q", name)
}
files[base] = true
if base == localName {
fileName = fi.Name()
}
}
if fileName == "" {
return "", errors.Errorf("No component name %q found", name)
}
return filepath.Join(ns.Dir(), fileName), nil
}
// Namespace is a component namespace.
type Namespace struct {
// Path is the path of the component namespace.
Path string
root string
fs afero.Fs
}
// ExtractNamespacedComponent extracts a namespace and a component from a path.
func ExtractNamespacedComponent(fs afero.Fs, root, path string) (Namespace, string) {
path, component := filepath.Split(path)
path = strings.TrimSuffix(path, "/")
ns := Namespace{Path: path, root: root, fs: fs}
return ns, component
}
// ParamsPath generates the path to params.libsonnet for a namespace.
func (n *Namespace) ParamsPath() string {
return filepath.Join(n.Dir(), paramsFile)
}
// ComponentPaths are the absolute paths to all the components in a namespace.
func (n *Namespace) ComponentPaths() ([]string, error) {
dir := n.Dir()
fis, err := afero.ReadDir(n.fs, dir)
if err != nil {
return nil, errors.Wrap(err, "read component dir")
}
var paths []string
for _, fi := range fis {
if fi.IsDir() {
continue
}
if strings.HasSuffix(fi.Name(), ".jsonnet") {
paths = append(paths, filepath.Join(dir, fi.Name()))
}
}
sort.Strings(paths)
return paths, nil
}
// Components returns the components in a namespace.
func (n *Namespace) Components() ([]string, error) {
paths, err := n.ComponentPaths()
if err != nil {
return nil, err
}
dir := filepath.Join(n.root, componentsRoot) + "/"
var names []string
for _, path := range paths {
name := strings.TrimPrefix(path, dir)
name = strings.TrimSuffix(name, filepath.Ext(name))
names = append(names, name)
}
return names, nil
}
// Dir is the absolute directory for a namespace.
func (n *Namespace) Dir() string {
path := []string{n.root, componentsRoot}
if n.Path != "" {
path = append(path, strings.Split(n.Path, "/")...)
}
return filepath.Join(path...)
}
// Namespaces returns all component namespaces
func Namespaces(fs afero.Fs, root string) ([]Namespace, error) {
componentRoot := filepath.Join(root, componentsRoot)
var namespaces []Namespace
err := afero.Walk(fs, componentRoot, func(path string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
if fi.IsDir() {
ok, err := isComponentDir(fs, path)
if err != nil {
return err
}
if ok {
nsPath := strings.TrimPrefix(path, componentRoot)
nsPath = strings.TrimPrefix(nsPath, "/")
ns := Namespace{Path: nsPath, fs: fs, root: root}
namespaces = append(namespaces, ns)
}
}
return nil
})
if err != nil {
return nil, errors.Wrap(err, "walk component path")
}
sort.Slice(namespaces, func(i, j int) bool {
return namespaces[i].Path < namespaces[j].Path
})
return namespaces, nil
}
func isComponentDir(fs afero.Fs, path string) (bool, error) {
files, err := afero.ReadDir(fs, path)
if err != nil {
return false, errors.Wrapf(err, "read files in %s", path)
}
for _, file := range files {
if file.Name() == paramsFile {
return true, nil
}
}
return false, nil
}
// AppSpecer is implemented by any value that has a AppSpec method. The AppSpec method is
// used to retrieve a ksonnet AppSpec.
type AppSpecer interface {
AppSpec() (*app.Spec, error)
}
// MakePathsByNamespace creates a map of component paths categorized by namespace.
func MakePathsByNamespace(fs afero.Fs, appSpecer AppSpecer, root, env string) (map[Namespace][]string, error) {
paths, err := MakePaths(fs, appSpecer, root, env)
if err != nil {
return nil, err
}
m := make(map[Namespace][]string)
for i := range paths {
prefix := root + "/components/"
if strings.HasSuffix(root, "/") {
prefix = root + "components/"
}
path := strings.TrimPrefix(paths[i], prefix)
ns, _ := ExtractNamespacedComponent(fs, root, path)
if _, ok := m[ns]; !ok {
m[ns] = make([]string, 0)
}
m[ns] = append(m[ns], paths[i])
}
return m, nil
}
// MakePaths creates a slice of component paths
func MakePaths(fs afero.Fs, appSpecer AppSpecer, root, env string) ([]string, error) {
cpl, err := newComponentPathLocator(fs, appSpecer, env)
if err != nil {
return nil, errors.Wrap(err, "create component path locator")
}
return cpl.Locate(root)
}
type componentPathLocator struct {
fs afero.Fs
envSpec *app.EnvironmentSpec
}
func newComponentPathLocator(fs afero.Fs, appSpecer AppSpecer, env string) (*componentPathLocator, error) {
if appSpecer == nil {
return nil, errors.New("appSpecer is nil")
}
if fs == nil {
return nil, errors.New("fs is nil")
}
appSpec, err := appSpecer.AppSpec()
if err != nil {
return nil, errors.Wrap(err, "lookup application spec")
}
envSpec, ok := appSpec.GetEnvironmentSpec(env)
if !ok {
return nil, errors.Errorf("can't find %s environment", env)
}
return &componentPathLocator{
fs: fs,
envSpec: envSpec,
}, nil
}
func (cpl *componentPathLocator) Locate(root string) ([]string, error) {
if len(cpl.envSpec.Targets) == 0 {
return cpl.defaultPaths(root)
}
var paths []string
for _, target := range cpl.envSpec.Targets {
childPaths, err := cpl.expandPath(root, target)
if err != nil {
return nil, errors.Wrapf(err, "unable to expand %s", target)
}
paths = append(paths, childPaths...)
}
sort.Strings(paths)
return paths, nil
}
// expandPath take a root and a target and returns all the jsonnet components in descendant paths.
func (cpl *componentPathLocator) expandPath(root, target string) ([]string, error) {
path := filepath.Join(root, componentsRoot, target)
fi, err := cpl.fs.Stat(path)
if err != nil {
return nil, err
}
var paths []string
walkFn := func(path string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
if !fi.IsDir() && isComponent(path) {
paths = append(paths, path)
}
return nil
}
if fi.IsDir() {
rootPath := filepath.Join(root, componentsRoot, fi.Name())
if err := afero.Walk(cpl.fs, rootPath, walkFn); err != nil {
return nil, errors.Wrapf(err, "search for components in %s", fi.Name())
}
} else if isComponent(fi.Name()) {
paths = append(paths, path)
}
return paths, nil
}
func (cpl *componentPathLocator) defaultPaths(root string) ([]string, error) {
var paths []string
walkFn := func(path string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
if !fi.IsDir() && isComponent(path) {
paths = append(paths, path)
}
return nil
}
componentRoot := filepath.Join(root, componentsRoot)
if err := afero.Walk(cpl.fs, componentRoot, walkFn); err != nil {
return nil, errors.Wrap(err, "search for components")
}
return paths, nil
}
// isComponent reports if a file is a component. Components have a `jsonnet` extension.
func isComponent(path string) bool {
return filepath.Ext(path) == ".jsonnet"
}
// Copyright 2017 The kubecfg authors
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package component
import (
"path/filepath"
"testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/ksonnet/ksonnet/metadata/app"
)
var (
existingPaths = []string{
"/app.yaml",
"/components/a.jsonnet",
"/components/b.jsonnet",
"/components/other",
"/components/params.libsonnet",
"/components/nested/a.jsonnet",
"/components/nested/params.libsonnet",
"/components/nested/very/deeply/c.jsonnet",
"/components/nested/very/deeply/params.libsonnet",
"/components/shallow/c.jsonnet",
"/components/shallow/params.libsonnet",
}
invalidPaths = []string{
"/app.yaml",
"/components/a.jsonnet",
"/components/a.txt",
}
)
type stubAppSpecer struct {
appSpec *app.Spec
err error
}
var _ AppSpecer = (*stubAppSpecer)(nil)
func newStubAppSpecer(appSpec *app.Spec) *stubAppSpecer {
return &stubAppSpecer{appSpec: appSpec}
}
func (s *stubAppSpecer) AppSpec() (*app.Spec, error) {
return s.appSpec, s.err
}
func makePaths(t *testing.T, fs afero.Fs, paths []string) {
for _, path := range paths {
dir := filepath.Dir(path)
err := fs.MkdirAll(dir, 0755)
require.NoError(t, err)
_, err = fs.Create(path)
require.NoError(t, err)
}
}
func TestPath(t *testing.T) {
cases := []struct {
name string
paths []string
in string
expected string
isErr bool
}{
{
name: "in root namespace",
paths: existingPaths,
in: "a",
expected: "/components/a.jsonnet",
},
{
name: "in nested namespace",
paths: existingPaths,
in: "nested/a",
expected: "/components/nested/a.jsonnet",
},
{
name: "not found",
paths: existingPaths,
in: "z",
isErr: true,
},
{
name: "invalid path",
paths: invalidPaths,
in: "a",
isErr: true,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
fs := afero.NewMemMapFs()
makePaths(t, fs, tc.paths)
path, err := Path(fs, "/", tc.in)
if tc.isErr {
require.Error(t, err)
} else {
require.NoError(t, err)
assert.Equal(t, tc.expected, path)
}
})
}
}
func TestExtractNamedspacedComponent(t *testing.T) {
cases := []struct {
name string
path string
nsPath string
component string
}{
{
name: "component in root namespace",
path: "my-deployment",
nsPath: "",
component: "my-deployment",
},
{
name: "component in root namespace",
path: "nested/my-deployment",
nsPath: "nested",
component: "my-deployment",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
fs := afero.NewMemMapFs()