Unverified Commit 83e92d64 authored by Bryan Liles's avatar Bryan Liles Committed by GitHub
Browse files

When importing YAML, extract objects into separate component files (#462)



* Create separate components for all items in YAML manifest
Signed-off-by: default avatarbryanl <bryanliles@gmail.com>

* Remove component index because it isn't needed

Since components are now one-per-file, remove the idea of indexes
because they are no longer needed.
Signed-off-by: default avatarbryanl <bryanliles@gmail.com>
parent 068ea643
......@@ -55,9 +55,6 @@ const (
OptionGlobal = "global"
// OptionGracePeriod is gracePeriod option.
OptionGracePeriod = "grace-period"
// OptionIndex is index option. Is used to target individual items in multi object
// components.
OptionIndex = "index"
// OptionLibName is libName.
OptionLibName = "lib-name"
// OptionName is name option.
......
......@@ -110,28 +110,24 @@ func (cl *ComponentList) listComponents(components []component.Component) {
func (cl *ComponentList) listComponentsWide(components []component.Component) error {
var rows [][]string
for _, c := range components {
summaries, err := c.Summarize()
summary, err := c.Summarize()
if err != nil {
return err
}
for _, summary := range summaries {
row := []string{
summary.ComponentName,
summary.Type,
summary.IndexStr,
summary.APIVersion,
summary.Kind,
summary.Name,
}
rows = append(rows, row)
row := []string{
summary.ComponentName,
summary.Type,
summary.APIVersion,
summary.Kind,
summary.Name,
}
rows = append(rows, row)
}
table := table.New(cl.out)
table.SetHeader([]string{"component", "type", "index", "apiversion", "kind", "name"})
table.SetHeader([]string{"component", "type", "apiversion", "kind", "name"})
table.AppendBulk(rows)
table.Render()
......
......@@ -68,9 +68,7 @@ func TestComponentList_wide(t *testing.T) {
module := ""
output := "wide"
summary := []component.Summary{
{ComponentName: "deployment"},
}
summary := component.Summary{ComponentName: "deployment"}
c := &cmocks.Component{}
c.On("Summarize").Return(summary, nil)
......
......@@ -17,7 +17,9 @@ package actions
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"mime"
"net/http"
"os"
......@@ -27,7 +29,9 @@ import (
"github.com/ksonnet/ksonnet/component"
"github.com/ksonnet/ksonnet/metadata/app"
ksparam "github.com/ksonnet/ksonnet/metadata/params"
"github.com/ksonnet/ksonnet/metadata/params"
"github.com/ksonnet/ksonnet/pkg/schema"
utilyaml "github.com/ksonnet/ksonnet/pkg/util/yaml"
"github.com/ksonnet/ksonnet/prototype"
"github.com/pkg/errors"
"github.com/spf13/afero"
......@@ -48,7 +52,8 @@ type Import struct {
app app.App
module string
path string
cm component.Manager
createComponentFn func(a app.App, name, text string, p params.Params, templateType prototype.TemplateType) (string, error)
}
// NewImport creates an instance of Import. `module` is the name of the component and
......@@ -61,7 +66,7 @@ func NewImport(m map[string]interface{}) (*Import, error) {
module: ol.LoadString(OptionModule),
path: ol.LoadString(OptionPath),
cm: component.DefaultManager,
createComponentFn: component.Create,
}
if ol.err != nil {
......@@ -130,8 +135,8 @@ func extractFilename(resp *http.Response) (string, error) {
filename := resp.Request.URL.Path
cd := resp.Header.Get("Content-Disposition")
if cd != "" {
if _, params, err := mime.ParseMediaType(cd); err == nil {
filename = params["filename"]
if _, contentDisposition, err := mime.ParseMediaType(cd); err == nil {
filename = contentDisposition["filename"]
}
}
......@@ -181,11 +186,6 @@ func (i *Import) handleLocal() error {
}
func (i *Import) importFile(fileName string) error {
var name bytes.Buffer
if i.module != "" {
name.WriteString(i.module + "/")
}
base := filepath.Base(fileName)
ext := filepath.Ext(base)
......@@ -194,6 +194,81 @@ func (i *Import) importFile(fileName string) error {
return errors.Wrap(err, "parse template type")
}
switch templateType {
default:
return errors.Errorf("unable to handle components of type %s", templateType)
case prototype.YAML:
return i.createYAML(fileName, base, ext)
case prototype.JSON, prototype.Jsonnet:
return i.createComponent(fileName, base, ext, templateType)
}
}
func (i *Import) createYAML(fileName, base, ext string) error {
readers, err := utilyaml.Decode(i.app.Fs(), fileName)
if err != nil {
return err
}
for _, r := range readers {
data, err := ioutil.ReadAll(r)
if err != nil {
return err
}
dataReader := bytes.NewReader(data)
ts, props, err := schema.ImportYaml(dataReader)
if err != nil {
if err == schema.ErrEmptyYAML {
continue
}
return err
}
val, err := props.Value([]string{"metadata", "name"})
if err != nil {
return err
}
name, ok := val.(string)
if !ok {
return errors.Errorf("unable to find metadata name of object in %s", fileName)
}
componentName := fmt.Sprintf("%s-%s", strings.ToLower(ts.Kind()), name)
if err = i.createComponentFromData(componentName, string(data), prototype.YAML); err != nil {
return err
}
}
return nil
}
func (i *Import) createComponentFromData(name, data string, templateType prototype.TemplateType) error {
componentParams := params.Params{}
switch i.module {
case "":
case "/":
name = fmt.Sprintf("/%s", name)
default:
name = fmt.Sprintf("%s/%s", i.module, name)
}
_, err := i.createComponentFn(i.app, name, data, componentParams, templateType)
if err != nil {
return errors.Wrap(err, "create component")
}
return nil
}
func (i *Import) createComponent(fileName, base, ext string, templateType prototype.TemplateType) error {
var name bytes.Buffer
if i.module != "" {
name.WriteString(i.module + "/")
}
name.WriteString(strings.TrimSuffix(base, ext))
contents, err := afero.ReadFile(i.app.Fs(), fileName)
......@@ -203,12 +278,13 @@ func (i *Import) importFile(fileName string) error {
sourcePath := filepath.Clean(name.String())
params := ksparam.Params{}
componentParams := params.Params{}
_, err = i.cm.CreateComponent(i.app, sourcePath, string(contents), params, templateType)
_, err = i.createComponentFn(i.app, sourcePath, string(contents), componentParams, templateType)
if err != nil {
return errors.Wrap(err, "create component")
}
return nil
}
......@@ -16,6 +16,7 @@
package actions
import (
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
......@@ -23,9 +24,10 @@ import (
"testing"
"time"
"github.com/stretchr/testify/mock"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
cmocks "github.com/ksonnet/ksonnet/component/mocks"
"github.com/ksonnet/ksonnet/metadata/app"
amocks "github.com/ksonnet/ksonnet/metadata/app/mocks"
"github.com/ksonnet/ksonnet/metadata/params"
"github.com/ksonnet/ksonnet/prototype"
......@@ -34,7 +36,11 @@ import (
func TestImport_http(t *testing.T) {
withApp(t, func(appMock *amocks.App) {
f, err := os.Open(filepath.Join("testdata", "import", "file.yaml"))
dataPath := filepath.Join("testdata", "import", "file.yaml")
serviceData, err := ioutil.ReadFile(dataPath)
require.NoError(t, err)
f, err := os.Open(dataPath)
require.NoError(t, err)
defer f.Close()
......@@ -56,21 +62,27 @@ func TestImport_http(t *testing.T) {
a, err := NewImport(in)
require.NoError(t, err)
cm := &cmocks.Manager{}
cm.On("CreateComponent", mock.Anything, "/manifest", "",
params.Params{}, prototype.YAML).Return("/", nil)
a.createComponentFn = func(_ app.App, name, text string, p params.Params, templateType prototype.TemplateType) (string, error) {
assert.Equal(t, "/service-my-service", name)
assert.Equal(t, string(serviceData), text)
assert.Equal(t, params.Params{}, p)
assert.Equal(t, prototype.YAML, templateType)
a.cm = cm
return "/", nil
}
err = a.Run()
require.NoError(t, err)
cm.AssertExpectations(t)
})
}
func TestImport_file(t *testing.T) {
withApp(t, func(appMock *amocks.App) {
dataPath := filepath.Join("testdata", "import", "file.yaml")
serviceData, err := ioutil.ReadFile(dataPath)
require.NoError(t, err)
module := "/"
path := "/file.yaml"
......@@ -85,11 +97,14 @@ func TestImport_file(t *testing.T) {
a, err := NewImport(in)
require.NoError(t, err)
cm := &cmocks.Manager{}
cm.On("CreateComponent", mock.Anything, "/file", "",
params.Params{}, prototype.YAML).Return("/", nil)
a.createComponentFn = func(_ app.App, name, text string, p params.Params, templateType prototype.TemplateType) (string, error) {
assert.Equal(t, "/service-my-service", name)
assert.Equal(t, string(serviceData), text)
assert.Equal(t, params.Params{}, p)
assert.Equal(t, prototype.YAML, templateType)
a.cm = cm
return "/", nil
}
err = a.Run()
require.NoError(t, err)
......@@ -98,6 +113,10 @@ func TestImport_file(t *testing.T) {
func TestImport_directory(t *testing.T) {
withApp(t, func(appMock *amocks.App) {
dataPath := filepath.Join("testdata", "import", "file.yaml")
serviceData, err := ioutil.ReadFile(dataPath)
require.NoError(t, err)
module := "/"
path := "/import"
......@@ -112,11 +131,14 @@ func TestImport_directory(t *testing.T) {
a, err := NewImport(in)
require.NoError(t, err)
cm := &cmocks.Manager{}
cm.On("CreateComponent", mock.Anything, "/file", "",
params.Params{}, prototype.YAML).Return("/", nil)
a.createComponentFn = func(_ app.App, name, text string, p params.Params, templateType prototype.TemplateType) (string, error) {
assert.Equal(t, "/service-my-service", name)
assert.Equal(t, string(serviceData), text)
assert.Equal(t, params.Params{}, p)
assert.Equal(t, prototype.YAML, templateType)
a.cm = cm
return "/", nil
}
err = a.Run()
require.NoError(t, err)
......@@ -137,6 +159,10 @@ func TestImport_invalid_file(t *testing.T) {
a, err := NewImport(in)
require.NoError(t, err)
a.createComponentFn = func(_ app.App, name, text string, p params.Params, templateType prototype.TemplateType) (string, error) {
return "", errors.New("invalid")
}
err = a.Run()
require.Error(t, err)
})
......
......@@ -43,7 +43,6 @@ type ParamDelete struct {
app app.App
name string
rawPath string
index int
global bool
envName string
......@@ -63,7 +62,6 @@ func NewParamDelete(m map[string]interface{}) (*ParamDelete, error) {
rawPath: ol.LoadString(OptionPath),
global: ol.LoadOptionalBool(OptionGlobal),
envName: ol.LoadOptionalString(OptionEnvName),
index: ol.LoadOptionalInt(OptionIndex),
deleteEnvFn: env.DeleteParam,
deleteEnvGlobalFn: env.UnsetGlobalParams,
......@@ -119,11 +117,7 @@ func (pd *ParamDelete) deleteLocal(path []string) error {
return errors.Wrap(err, "could not find component")
}
options := component.ParamOptions{
Index: pd.index,
}
if err := c.DeleteParam(path, options); err != nil {
if err := c.DeleteParam(path); err != nil {
return errors.Wrap(err, "delete param")
}
......
......@@ -32,7 +32,7 @@ func TestParamDelete(t *testing.T) {
path := "replicas"
c := &cmocks.Component{}
c.On("DeleteParam", []string{"replicas"}, component.ParamOptions{}).Return(nil)
c.On("DeleteParam", []string{"replicas"}).Return(nil)
in := map[string]interface{}{
OptionApp: appMock,
......@@ -52,33 +52,6 @@ func TestParamDelete(t *testing.T) {
})
}
func TestParamDelete_index(t *testing.T) {
withApp(t, func(appMock *amocks.App) {
componentName := "deployment"
path := "replicas"
c := &cmocks.Component{}
c.On("DeleteParam", []string{"replicas"}, component.ParamOptions{Index: 1}).Return(nil)
in := map[string]interface{}{
OptionApp: appMock,
OptionName: componentName,
OptionPath: path,
OptionIndex: 1,
}
a, err := NewParamDelete(in)
require.NoError(t, err)
a.resolvePathFn = func(app.App, string) (component.Module, component.Component, error) {
return nil, c, nil
}
err = a.Run()
require.NoError(t, err)
})
}
func TestParamDelete_global(t *testing.T) {
withApp(t, func(appMock *amocks.App) {
module := "/"
......
......@@ -134,9 +134,9 @@ func (pl *ParamList) collectParams(module component.Module) ([]component.ModuleP
func (pl *ParamList) print(params []component.ModuleParameter) error {
table := table.New(pl.out)
table.SetHeader([]string{"COMPONENT", "INDEX", "PARAM", "VALUE"})
table.SetHeader([]string{"COMPONENT", "PARAM", "VALUE"})
for _, data := range params {
table.Append([]string{data.Component, data.Index, data.Key, data.Value})
table.Append([]string{data.Component, data.Key, data.Value})
}
return table.Render()
......
......@@ -42,7 +42,6 @@ type ParamSet struct {
name string
rawPath string
rawValue string
index int
global bool
envName string
......@@ -63,7 +62,6 @@ func NewParamSet(m map[string]interface{}) (*ParamSet, error) {
rawValue: ol.LoadString(OptionValue),
global: ol.LoadOptionalBool(OptionGlobal),
envName: ol.LoadOptionalString(OptionEnvName),
index: ol.LoadOptionalInt(OptionIndex),
getModuleFn: component.GetModule,
resolvePathFn: component.ResolvePath,
......@@ -124,10 +122,7 @@ func (ps *ParamSet) setLocal(path []string, value interface{}) error {
return errors.Wrap(err, "could not find component")
}
options := component.ParamOptions{
Index: ps.index,
}
if err := c.SetParam(path, value, options); err != nil {
if err := c.SetParam(path, value); err != nil {
return errors.Wrap(err, "set param")
}
......
......@@ -33,7 +33,7 @@ func TestParamSet(t *testing.T) {
value := "3"
c := &cmocks.Component{}
c.On("SetParam", []string{"replicas"}, 3, component.ParamOptions{}).Return(nil)
c.On("SetParam", []string{"replicas"}, 3).Return(nil)
in := map[string]interface{}{
OptionApp: appMock,
......@@ -54,35 +54,6 @@ func TestParamSet(t *testing.T) {
})
}
func TestParamSet_index(t *testing.T) {
withApp(t, func(appMock *amocks.App) {
componentName := "deployment"
path := "replicas"
value := "3"
c := &cmocks.Component{}
c.On("SetParam", []string{"replicas"}, 3, component.ParamOptions{Index: 1}).Return(nil)
in := map[string]interface{}{
OptionApp: appMock,
OptionName: componentName,
OptionPath: path,
OptionValue: value,
OptionIndex: 1,
}
a, err := NewParamSet(in)
require.NoError(t, err)
a.resolvePathFn = func(app.App, string) (component.Module, component.Component, error) {
return nil, c, nil
}
err = a.Run()
require.NoError(t, err)
})
}
func TestParamSet_global(t *testing.T) {
withApp(t, func(appMock *amocks.App) {
module := "/"
......
COMPONENT TYPE INDEX APIVERSION KIND NAME
========= ==== ===== ========== ==== ====
COMPONENT TYPE APIVERSION KIND NAME
========= ==== ========== ==== ====
deployment
kind: Service
apiVersion: v1
metadata:
name: my-service
spec:
selector:
app: MyApp
ports:
- protocol: TCP
port: 80
targetPort: 9376
COMPONENT INDEX PARAM VALUE
========= ===== ===== =====
deployment 0 key "value"
COMPONENT PARAM VALUE
========= ===== =====
deployment key "value"
COMPONENT INDEX PARAM VALUE
========= ===== ===== =====
deployment 0 key "value"
COMPONENT PARAM VALUE
========= ===== =====
deployment key "value"
COMPONENT INDEX PARAM VALUE
========= ===== ===== =====
deployment 0 key "value"
COMPONENT PARAM VALUE
========= ===== =====
deployment key "value"
......@@ -29,7 +29,6 @@ const (
flagFilename = "filename"
flagGcTag = "gc-tag"
flagGracePeriod = "grace-period"
flagIndex = "index"
flagJpath = "jpath"
flagModule = "module"
flagNamespace = "namespace"
......@@ -49,7 +48,6 @@ const (
shortComponent = "c"
shortFilename = "f"
shortFormat = "o"
shortIndex = "i"
shortOutput = "o"
shortOverride = "o"
)
......@@ -23,8 +23,7 @@ import (
)
var (
vParamDeleteEnv = "param-delete-env"
vParamDeleteIndex = "param-delete-index"
vParamDeleteEnv = "param-delete-env"
)
var paramDeleteCmd = &cobra.Command{
......@@ -49,7 +48,6 @@ var paramDeleteCmd = &cobra.Command{
actions.OptionName: name,
actions.OptionPath: path,
actions.OptionEnvName: viper.GetString(vParamDeleteEnv),
actions.OptionIndex: viper.GetInt(vParamDeleteIndex),
}
return runAction(actionParamDelete, m)
......@@ -78,6 +76,4 @@ func init() {
paramDeleteCmd.Flags().String(flagEnv, "", "Specify environment to delete parameter from")
viper.BindPFlag(vParamDeleteEnv, paramDeleteCmd.Flags().Lookup(flagEnv))
paramDeleteCmd.Flags().IntP(flagIndex, shortIndex, 0, "Index in manifest")
viper.BindPFlag(vParamDeleteIndex, paramDeleteCmd.Flags().Lookup(flagIndex))
}
......@@ -32,7 +32,6 @@ func Test_paramDeleteCmd(t *testing.T) {
actions.OptionName: "component-name",
actions.OptionPath: "param-name",
actions.OptionEnvName: "",
actions.OptionIndex: 0,
},
},
{
......@@ -44,7 +43,6 @@ func Test_paramDeleteCmd(t *testing.T) {
actions.OptionName: "",
actions.OptionPath: "param-name",
actions.OptionEnvName: "default",
actions.OptionIndex: 0,
},
},
}
......
......@@ -23,8 +23,7 @@ import (
)
var (
vParamSetEnv = "param-set-env"
vParamSetIndex = "param-set-index"
vParamSetEnv = "param-set-env"
)
var paramSetCmd = &cobra.Command{
......@@ -53,7 +52,6 @@ var paramSetCmd = &cobra.Command{
actions.OptionPath: path,
actions.OptionValue: value,