Merge pull request #383 from bryanl/add-component

Update package caching

actions/pkg_describe.go

+// Copyright 2018 The ksonnet authors
+//
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+
+package actions
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/ksonnet/ksonnet/metadata/app"
+	"github.com/ksonnet/ksonnet/pkg/pkg"
+	"github.com/ksonnet/ksonnet/pkg/registry"
+)
+
+// RunPkgDescribe runs `pkg install`
+func RunPkgDescribe(ksApp app.App, pkgName string) error {
+	pd, err := NewPkgDescribe(ksApp, pkgName)
+	if err != nil {
+		return err
+	}
+
+	return pd.Run()
+}
+
+// PkgDescribe lists namespaces.
+type PkgDescribe struct {
+	app            app.App
+	pkgName        string
+	out            io.Writer
+	libPartFn      func(app.App, string) (*pkg.Package, error)
+	registryPartFn func(app.App, string) (*pkg.Package, error)
+}
+
+// NewPkgDescribe creates an instance of PkgDescribe.
+func NewPkgDescribe(ksApp app.App, pkgName string) (*PkgDescribe, error) {
+	pd := &PkgDescribe{
+		app:            ksApp,
+		pkgName:        pkgName,
+		out:            os.Stdout,
+		libPartFn:      pkg.Find,
+		registryPartFn: registry.Package,
+	}
+
+	return pd, nil
+}
+
+// Run lists namespaces.
+func (pd *PkgDescribe) Run() error {
+	d, err := pkg.ParseName(pd.pkgName)
+	if err != nil {
+		return err
+	}
+
+	var p *pkg.Package
+	if d.Registry == "" {
+		p, err = pd.libPartFn(pd.app, pd.pkgName)
+		if err != nil {
+			return err
+		}
+	} else {
+		p, err = pd.registryPartFn(pd.app, pd.pkgName)
+		if err != nil {
+			return err
+		}
+	}
+
+	fmt.Fprintln(pd.out, `LIBRARY NAME:`)
+	fmt.Fprintln(pd.out, p.Name)
+	fmt.Fprintln(pd.out)
+	fmt.Fprintln(pd.out, `DESCRIPTION:`)
+	fmt.Fprintln(pd.out, p.Description)
+	fmt.Fprintln(pd.out)
+	fmt.Fprintln(pd.out, `PROTOTYPES:`)
+
+	for _, proto := range p.Prototypes {
+		fmt.Fprintf(pd.out, "  %s\n", proto.Name)
+	}
+
+	return nil
+}

actions/pkg_describe_test.go

+// Copyright 2018 The ksonnet authors
+//
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+
+package actions
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/ksonnet/ksonnet/metadata/app"
+	amocks "github.com/ksonnet/ksonnet/metadata/app/mocks"
+	"github.com/ksonnet/ksonnet/pkg/pkg"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPkgDescribe_library(t *testing.T) {
+	withApp(t, func(appMock *amocks.App) {
+		libaries := app.LibraryRefSpecs{
+			"apache": &app.LibraryRefSpec{},
+		}
+		appMock.On("Libraries").Return(libaries, nil)
+
+		a, err := NewPkgDescribe(appMock, "apache")
+		require.NoError(t, err)
+
+		var buf bytes.Buffer
+		a.out = &buf
+
+		a.libPartFn = func(a app.App, name string) (*pkg.Package, error) {
+			p := &pkg.Package{
+				Name:        "apache",
+				Description: "description",
+			}
+
+			return p, nil
+		}
+
+		err = a.Run()
+		require.NoError(t, err)
+
+		assertOutput(t, "pkg/describe/output.txt", buf.String())
+	})
+}
+
+func TestPkgDescribe_registry(t *testing.T) {
+	withApp(t, func(appMock *amocks.App) {
+		registries := app.RegistryRefSpecs{
+			"incubator": &app.RegistryRefSpec{},
+		}
+		appMock.On("Registries").Return(registries, nil)
+
+		a, err := NewPkgDescribe(appMock, "incubator/apache")
+		require.NoError(t, err)
+
+		var buf bytes.Buffer
+		a.out = &buf
+
+		a.registryPartFn = func(a app.App, name string) (*pkg.Package, error) {
+			p := &pkg.Package{
+				Name:        "apache",
+				Description: "description",
+			}
+
+			return p, nil
+		}
+
+		err = a.Run()
+		require.NoError(t, err)
+
+		assertOutput(t, "pkg/describe/output.txt", buf.String())
+	})
+}

actions/pkg_install.go

@@ -16,6 +16,7 @@
 package actions
 
 import (
+	"github.com/davecgh/go-spew/spew"
 	"github.com/ksonnet/ksonnet/metadata/app"
 	"github.com/ksonnet/ksonnet/pkg/pkg"
 	"github.com/ksonnet/ksonnet/pkg/registry"
@@ -75,6 +76,8 @@ func (pi *PkgInstall) Run() error {
 		return err
 	}
 
+	spew.Dump(d)
+
 	return pi.depCacher(pi.app, d, customName)
 }
 

actions/testdata/pkg/describe/output.txt

+LIBRARY NAME:
+apache
+
+DESCRIPTION:
+description
+
+PROTOTYPES:

cmd/pkg.go

@@ -17,11 +17,8 @@ package cmd
 
 import (
 	"fmt"
-	"os"
 	"strings"
 
-	"github.com/ksonnet/ksonnet/metadata"
-	"github.com/ksonnet/ksonnet/pkg/parts"
 	"github.com/spf13/cobra"
 )
 
@@ -39,7 +36,6 @@ var errInvalidSpec = fmt.Errorf("Command 'pkg install' requires a single argumen
 
 func init() {
 	RootCmd.AddCommand(pkgCmd)
-	pkgCmd.AddCommand(pkgDescribeCmd)
 }
 
 var pkgCmd = &cobra.Command{
@@ -79,79 +75,6 @@ See the annotated file tree below, as an example:
 	},
 }
 
-var pkgDescribeCmd = &cobra.Command{
-	Use:   "describe [<registry-name>/]<package-name>",
-	Short: pkgShortDesc["describe"],
-	RunE: func(cmd *cobra.Command, args []string) error {
-		if len(args) != 1 {
-			return fmt.Errorf("Command 'pkg describe' requires a package name\n\n%s", cmd.UsageString())
-		}
-
-		registryName, libID, err := parsePkgSpec(args[0])
-		if err == errInvalidSpec {
-			registryName = ""
-			libID = args[0]
-		} else if err != nil {
-			return err
-		}
-
-		cwd, err := os.Getwd()
-		if err != nil {
-			return err
-		}
-
-		manager, err := metadata.Find(cwd)
-		if err != nil {
-			return err
-		}
-
-		var lib *parts.Spec
-		if registryName == "" {
-			lib, err = manager.GetDependency(libID)
-			if err != nil {
-				return err
-			}
-		} else {
-			lib, err = manager.GetPackage(registryName, libID)
-			if err != nil {
-				return err
-			}
-		}
-
-		fmt.Println(`LIBRARY NAME:`)
-		fmt.Println(lib.Name)
-		fmt.Println()
-		fmt.Println(`DESCRIPTION:`)
-		fmt.Println(lib.Description)
-		fmt.Println()
-		fmt.Println(`PROTOTYPES:`)
-		for _, proto := range lib.Prototypes {
-			fmt.Printf("  %s\n", proto)
-		}
-		fmt.Println()
-
-		return nil
-	},
-
-	Long: `
-The ` + "`describe`" + ` command outputs documentation for a package that is available
-(e.g. downloaded) in the current ksonnet application. (This must belong to an already
-known ` + "`<registry-name>`" + ` like *incubator*). The output includes:
-
-1. The library name
-2. A brief description provided by the library authors
-3. A list of available prototypes provided by the library
-
-### Related Commands
-
-* ` + "`ks pkg list` " + `— ` + pkgShortDesc["list"] + `
-* ` + "`ks prototype describe` " + `— ` + protoShortDesc["describe"] + `
-* ` + "`ks generate` " + `— ` + protoShortDesc["use"] + `
-
-### Syntax
-`,
-}
-
 func parsePkgSpec(spec string) (registry, libID string, err error) {
 	split := strings.SplitN(spec, "/", 2)
 	if len(split) < 2 {

cmd/pkg_describe.go

+// Copyright 2018 The ksonnet authors
+//
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/ksonnet/ksonnet/actions"
+	"github.com/spf13/cobra"
+)
+
+var pkgDescribeCmd = &cobra.Command{
+	Use:   "describe [<registry-name>/]<package-name>",
+	Short: pkgShortDesc["describe"],
+	RunE: func(cmd *cobra.Command, args []string) error {
+		if len(args) != 1 {
+			return fmt.Errorf("Command 'pkg describe' requires a package name\n\n%s", cmd.UsageString())
+		}
+
+		return actions.RunPkgDescribe(ka, args[0])
+	},
+
+	Long: `
+The ` + "`describe`" + ` command outputs documentation for a package that is available
+(e.g. downloaded) in the current ksonnet application. (This must belong to an already
+known ` + "`<registry-name>`" + ` like *incubator*). The output includes:
+
+1. The library name
+2. A brief description provided by the library authors
+3. A list of available prototypes provided by the library
+
+### Related Commands
+
+* ` + "`ks pkg list` " + `— ` + pkgShortDesc["list"] + `
+* ` + "`ks prototype describe` " + `— ` + protoShortDesc["describe"] + `
+* ` + "`ks generate` " + `— ` + protoShortDesc["use"] + `
+
+### Syntax
+`,
+}
+
+func init() {
+	pkgCmd.AddCommand(pkgDescribeCmd)
+}

cmd/root.go

@@ -351,59 +351,6 @@ func (te *cmdObjExpander) Expand() ([]*unstructured.Unstructured, error) {
 
 	p := pipeline.New(ksApp, te.config.env)
 	return p.Objects(te.config.components)
-
-	// //
-	// // Set up the template expander to be able to expand the ksonnet application.
-	// //
-
-	// envPath, vendorPath := manager.LibPaths()
-	// libPath, mainPath, paramsPath, err := manager.EnvPaths(te.config.env)
-	// if err != nil {
-	// 	return nil, err
-	// }
-
-	// expander.FlagJpath = append([]string{string(vendorPath), string(libPath), string(envPath)}, expander.FlagJpath...)
-
-	// namespacedComponentPaths, err := component.MakePathsByNamespace(app, te.config.env)
-	// if err != nil {
-	// 	return nil, errors.Wrap(err, "component paths")
-	// }
-
-	// //
-	// // Set up ExtCodes to resolve runtime variables such as the environment namespace.
-	// //
-
-	// envSpec, err := importEnv(manager, te.config.env)
-	// if err != nil {
-	// 	return nil, err
-	// }
-
-	// baseCodes := expander.ExtCodes
-
-	// params := importParams(paramsPath)
-
-	// slUnstructured := make([]*unstructured.Unstructured, 0)
-	// for ns, componentPaths := range namespacedComponentPaths {
-
-	// 	baseObj, err := constructBaseObj(componentPaths, te.config.components)
-	// 	if err != nil {
-	// 		return nil, errors.Wrap(err, "construct base object")
-	// 	}
-
-	// 	//
-	// 	// Expand the ksonnet app as rendered for environment `env`.
-	// 	//
-	// 	expander.ExtCodes = append([]string{baseObj, params, envSpec}, baseCodes...)
-	// 	u, err := expander.Expand([]string{string(mainPath)})
-	// 	if err != nil {
-	// 		return nil, errors.Wrapf(err, "generate objects for namespace %s", ns.Name())
-	// 	}
-
-	// 	slUnstructured = append(slUnstructured, u...)
-	// }
-
-	// return slUnstructured, nil
-
 }
 
 // constructBaseObj constructs the base Jsonnet object that represents k-v

component/jsonnet.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ksonnet/ksonnet/pkg/params"
 	"github.com/ksonnet/ksonnet/pkg/util/k8s"
 	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/afero"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -145,6 +146,7 @@ func (j *Jsonnet) Objects(paramsStr, envName string) ([]*unstructured.Unstructur
 
 	vm := jsonnet.MakeVM()
 	vm.Importer(importer)
+	log.Debugf("%s convert to jsonnet: setting %q to %q", j.Name(true), "__ksonnet/params", paramsStr)
 	vm.ExtCode("__ksonnet/params", paramsStr)
 
 	snippet, err := afero.ReadFile(j.app.Fs(), j.source)

component/manager.go

@@ -20,7 +20,6 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/davecgh/go-spew/spew"
 	"github.com/ksonnet/ksonnet/metadata/app"
 	param "github.com/ksonnet/ksonnet/metadata/params"
 	"github.com/ksonnet/ksonnet/prototype"
@@ -105,8 +104,6 @@ func (dm *defaultManager) ResolvePath(ksApp app.App, path string) (Namespace, Co
 		return nil, nil, err
 	}
 
-	spew.Dump(nsName)
-
 	ns, err := dm.Namespace(ksApp, nsName)
 	if err != nil {
 		return nil, nil, err

e2e/app.go

@@ -63,6 +63,13 @@ func (a *app) paramList(args ...string) *output {
 	return o
 }
 
+func (a *app) pkgInstall(partName string) *output {
+	o := a.runKs("pkg", "install", partName)
+	assertExitStatus(o, 0)
+
+	return o
+}
+
 func (a *app) pkgList() *output {
 	o := a.runKs("pkg", "list")
 	assertExitStatus(o, 0)

e2e/pkg_test.go

@@ -42,13 +42,22 @@ var _ = Describe("ks pkg", func() {
 	})
 
 	Describe("describe", func() {
-		Context("incubator/apache", func() {
-			It("describes the package", func() {
+		Context("registry incubator/apache", func() {
+			It("describes the registry package", func() {
 				o := a.runKs("pkg", "describe", "incubator/apache")
 				assertExitStatus(o, 0)
 				assertOutput("pkg/describe/output.txt", o.stdout)
 			})
 		})
+		Context("library apache", func() {
+			It("describes the library package", func() {
+				a.pkgInstall("incubator/apache")
+
+				o := a.runKs("pkg", "describe", "apache")
+				assertExitStatus(o, 0)
+				assertOutput("pkg/describe/output.txt", o.stdout)
+			})
+		})
 	})
 
 	Describe("install", func() {

e2e/testdata/output/pkg/describe/output.txt

@@ -5,4 +5,3 @@ DESCRIPTION:
 Apache Ksonnet mixin library contains a simple prototype with pre-configured components to help you deploy a Apache HTTP Server app to a Kubernetes cluster with ease.
 
 PROTOTYPES:
-

e2e/testdata/output/version/default.txt

 ksonnet version: (dev build)
-jsonnet version: v0.9.5
+jsonnet version: v0.10.0
 client-go version: 1.8

metadata/app/app_test.go

+// Copyright 2018 The ksonnet authors
+//
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+
 package app
 
 import (

pkg/pipeline/pipeline.go

@@ -19,6 +19,7 @@ import (
 	"bytes"
 	"io"
 	"regexp"
+	"strings"
 
 	jsonnet "github.com/google/go-jsonnet"
 	"github.com/ksonnet/ksonnet/component"
@@ -117,6 +118,13 @@ func (p *Pipeline) Objects(filter []string) ([]*unstructured.Unstructured, error
 		return nil, err
 	}
 
+	var names []string
+	for _, n := range namespaces {
+		names = append(names, n.Name())
+	}
+
+	logrus.Debugf("pipeline: build objects for namespaces: %s", strings.Join(names, ", "))
+
 	objects := make([]*unstructured.Unstructured, 0)
 	for _, ns := range namespaces {
 		paramsStr, err := p.EnvParameters(ns.Name())
@@ -129,8 +137,15 @@ func (p *Pipeline) Objects(filter []string) ([]*unstructured.Unstructured, error
 			return nil, err
 		}
 
+		names = make([]string, 0)
 		for _, c := range components {
-			o, err := c.Objects(paramsStr, p.envName)
+			names = append(names, c.Name(true))
+		}
+
+		logrus.Debugf("pipeline: components for %s: %s", ns.Name(), strings.Join(names, ", "))
+
+		for i := range components {
+			o, err := components[i].Objects(paramsStr, p.envName)
 			if err != nil {
 				return nil, err
 			}

pkg/pipeline/pipeline_test.go

@@ -114,6 +114,7 @@ func TestPipeline_Objects(t *testing.T) {
 
 		cpnt := &cmocks.Component{}
 		cpnt.On("Objects", mock.Anything, "default").Return(u, nil)
+		cpnt.On("Name", true).Return("name")
 		components := []component.Component{cpnt}
 
 		ns := component.NewNamespace(p.app, "/")
@@ -139,6 +140,7 @@ func TestPipeline_YAML(t *testing.T) {
 
 		cpnt := &cmocks.Component{}
 		cpnt.On("Objects", mock.Anything, "default").Return(u, nil)
+		cpnt.On("Name", true).Return("name")
 		components := []component.Component{cpnt}
 
 		ns := component.NewNamespace(p.app, "/")

pkg/pkg/name.go

@@ -17,12 +17,15 @@ package pkg
 
 import (
 	"fmt"
+	"regexp"
 	"strings"
-
-	"github.com/pkg/errors"
 )
 
-var errInvalidSpec = fmt.Errorf("package name should be in the form `<registry>/<library>@<version>`")
+var (
+	errInvalidSpec = fmt.Errorf("package name should be in the form `<registry>/<library>@<version>`")
+
+	reDescriptor = regexp.MustCompile(`^([A-Za-z0-9\-]+)(\/[^@]+)?(@[^@]+)?$`)
+)
 
 // Descriptor describes a package.
 type Descriptor struct {
@@ -33,25 +36,21 @@ type Descriptor struct {
 
 // ParseName parses a package name into its components
 func ParseName(name string) (Descriptor, error) {
-	split := strings.SplitN(name, "/", 2)
-	if len(split) < 2 {
+	matches := reDescriptor.FindStringSubmatch(name)
+	if len(matches) == 0 {
 		return Descriptor{}, errInvalidSpec
 	}
 
-	registryName := split[0]
-	partName := strings.SplitN(split[1], "@", 2)[0]
-
-	split = strings.Split(name, "@")
-	if len(split) > 2 {
-		return Descriptor{}, errors.Errorf("symbol '@' is only allowed once, at the end of the argument of the form <registry>/<library>@<version>")
-	}
-	var version string
-	if len(split) == 2 {
-		version = split[1]