From f2955fedd04e50124316f660e9bf17f3460c4fb8 Mon Sep 17 00:00:00 2001
From: Angus Lees <gus@inodes.org>
Date: Fri, 14 Jul 2017 17:04:09 +1000
Subject: [PATCH] Implement garbage collection
Fixes #15
---
cmd/root.go | 12 ++-
cmd/update.go | 225 +++++++++++++++++++++++++++++++++++++++++++--
cmd/update_test.go | 81 ++++++++++++++++
utils/meta.go | 11 +++
4 files changed, 320 insertions(+), 9 deletions(-)
create mode 100644 cmd/update_test.go
diff --git a/cmd/root.go b/cmd/root.go
index 152847ce..0f7e9842 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -31,8 +31,10 @@ import (
"github.com/spf13/cobra"
jsonnet "github.com/strickyak/jsonnet_cgo"
"golang.org/x/crypto/ssh/terminal"
+ "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+ "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/discovery"
"k8s.io/client-go/dynamic"
@@ -375,8 +377,8 @@ func serverResourceForGroupVersionKind(disco discovery.DiscoveryInterface, gvk s
return nil, fmt.Errorf("Server is unable to handle %s", gvk)
}
-func clientForResource(pool dynamic.ClientPool, disco discovery.DiscoveryInterface, obj *unstructured.Unstructured, defNs string) (*dynamic.ResourceClient, error) {
- gvk := obj.GroupVersionKind()
+func clientForResource(pool dynamic.ClientPool, disco discovery.DiscoveryInterface, obj runtime.Object, defNs string) (*dynamic.ResourceClient, error) {
+ gvk := obj.GetObjectKind().GroupVersionKind()
client, err := pool.ClientForGroupVersionKind(gvk)
if err != nil {
@@ -388,7 +390,11 @@ func clientForResource(pool dynamic.ClientPool, disco discovery.DiscoveryInterfa
return nil, err
}
- namespace := obj.GetNamespace()
+ meta, err := meta.Accessor(obj)
+ if err != nil {
+ return nil, err
+ }
+ namespace := meta.GetNamespace()
if namespace == "" {
namespace = defNs
}
diff --git a/cmd/update.go b/cmd/update.go
index f5b5ff40..b3787edc 100644
--- a/cmd/update.go
+++ b/cmd/update.go
@@ -23,20 +23,47 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"k8s.io/apimachinery/pkg/api/errors"
+ "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/diff"
+ "k8s.io/apimachinery/pkg/util/sets"
+ "k8s.io/client-go/discovery"
+ "k8s.io/client-go/dynamic"
"github.com/ksonnet/kubecfg/utils"
)
const (
flagCreate = "create"
+ flagSkipGc = "skip-gc"
+ flagGcTag = "gc-tag"
+ flagDryRun = "dry-run"
+
+ // AnnotationGcTag annotation that triggers
+ // garbage collection. Objects with value equal to
+ // command-line flag that are *not* in config will be deleted.
+ AnnotationGcTag = "kubecfg.ksonnet.io/garbage-collect-tag"
+
+ // AnnotationGcStrategy controls gc logic. Current values:
+ // `auto` (default if absent) - do garbage collection
+ // `ignore` - never garbage collect this object
+ AnnotationGcStrategy = "kubecfg.ksonnet.io/garbage-collect-strategy"
+
+ // GcStrategyAuto is the default automatic gc logic
+ GcStrategyAuto = "auto"
+ // GcStrategyIgnore means this object should be ignored by garbage collection
+ GcStrategyIgnore = "ignore"
)
func init() {
RootCmd.AddCommand(updateCmd)
updateCmd.PersistentFlags().Bool(flagCreate, true, "Create missing resources")
+ updateCmd.PersistentFlags().Bool(flagSkipGc, false, "Don't perform garbage collection, even with --"+flagGcTag)
+ updateCmd.PersistentFlags().String(flagGcTag, "", "Add this tag to updated objects, and garbage collect existing objects with this tag and not in config")
+ updateCmd.PersistentFlags().Bool(flagDryRun, false, "Perform only read-only operations")
}
var updateCmd = &cobra.Command{
@@ -50,7 +77,17 @@ var updateCmd = &cobra.Command{
return err
}
- objs, err := readObjs(cmd, args)
+ gcTag, err := flags.GetString(flagGcTag)
+ if err != nil {
+ return err
+ }
+
+ skipGc, err := flags.GetBool(flagSkipGc)
+ if err != nil {
+ return err
+ }
+
+ dryRun, err := flags.GetBool(flagDryRun)
if err != nil {
return err
}
@@ -65,13 +102,29 @@ var updateCmd = &cobra.Command{
return err
}
+ objs, err := readObjs(cmd, args)
+ if err != nil {
+ return err
+ }
+
+ dryRunText := ""
+ if dryRun {
+ dryRunText = " (dry-run)"
+ }
+
sort.Sort(utils.DependencyOrder(objs))
+ seenUids := sets.NewString()
+
for _, obj := range objs {
+ if gcTag != "" {
+ utils.SetMetaDataAnnotation(obj, AnnotationGcTag, gcTag)
+ }
+
desc := fmt.Sprintf("%s/%s", obj.GetKind(), fqName(obj))
- log.Info("Updating ", desc)
+ log.Info("Updating ", desc, dryRunText)
- c, err := clientForResource(clientpool, disco, obj, defaultNs)
+ rc, err := clientForResource(clientpool, disco, obj, defaultNs)
if err != nil {
return err
}
@@ -80,10 +133,22 @@ var updateCmd = &cobra.Command{
if err != nil {
return err
}
- newobj, err := c.Patch(obj.GetName(), types.MergePatchType, asPatch)
+ var newobj metav1.Object
+ if !dryRun {
+ newobj, err = rc.Patch(obj.GetName(), types.MergePatchType, asPatch)
+ log.Debug("Patch(%s) returned (%v, %v)", obj.GetName(), newobj, err)
+ } else {
+ newobj, err = rc.Get(obj.GetName())
+ }
if create && errors.IsNotFound(err) {
- log.Info(" Creating non-existent ", desc)
- newobj, err = c.Create(obj)
+ log.Info(" Creating non-existent ", desc, dryRunText)
+ if !dryRun {
+ newobj, err = rc.Create(obj)
+ log.Debug("Create(%s) returned (%v, %v)", obj.GetName(), newobj, err)
+ } else {
+ newobj = obj
+ err = nil
+ }
}
if err != nil {
// TODO: retry
@@ -91,6 +156,43 @@ var updateCmd = &cobra.Command{
}
log.Debug("Updated object: ", diff.ObjectDiff(obj, newobj))
+
+ // Some objects appear under multiple kinds
+ // (eg: Deployment is both extensions/v1beta1
+ // and apps/v1beta1). UID is the only stable
+ // identifier that links these two views of
+ // the same object.
+ seenUids.Insert(string(newobj.GetUID()))
+ }
+
+ if gcTag != "" && !skipGc {
+ version, err := utils.FetchVersion(disco)
+ if err != nil {
+ return err
+ }
+
+ err = walkObjects(clientpool, disco, metav1.ListOptions{}, func(o runtime.Object) error {
+ meta, err := meta.Accessor(o)
+ if err != nil {
+ return err
+ }
+ gvk := o.GetObjectKind().GroupVersionKind()
+ desc := fmt.Sprintf("%s/%s (%s)", gvk.Kind, fqName(meta), gvk.GroupVersion())
+ log.Debugf("Considering %v for gc", desc)
+ if eligibleForGc(meta, gcTag) && !seenUids.Has(string(meta.GetUID())) {
+ log.Info("Garbage collecting ", desc, dryRunText)
+ if !dryRun {
+ err := gcDelete(clientpool, disco, &version, o)
+ if err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ return err
+ }
}
return nil
@@ -103,3 +205,114 @@ func fqName(o metav1.Object) string {
}
return fmt.Sprintf("%s.%s", o.GetNamespace(), o.GetName())
}
+
+func stringListContains(list []string, value string) bool {
+ for _, item := range list {
+ if item == value {
+ return true
+ }
+ }
+ return false
+}
+
+func gcDelete(clientpool dynamic.ClientPool, disco discovery.DiscoveryInterface, version *utils.ServerVersion, o runtime.Object) error {
+ obj, err := meta.Accessor(o)
+ if err != nil {
+ return fmt.Errorf("Unexpected object type: %s", err)
+ }
+
+ uid := obj.GetUID()
+ desc := fmt.Sprintf("%s/%s", o.GetObjectKind().GroupVersionKind().Kind, fqName(obj))
+
+ deleteOpts := metav1.DeleteOptions{
+ Preconditions: &metav1.Preconditions{UID: &uid},
+ }
+ if version.Compare(1, 6) < 0 {
+ // 1.5.x option
+ boolFalse := false
+ deleteOpts.OrphanDependents = &boolFalse
+ } else {
+ // 1.6.x option (NB: Background is broken)
+ fg := metav1.DeletePropagationForeground
+ deleteOpts.PropagationPolicy = &fg
+ }
+
+ c, err := clientForResource(clientpool, disco, o, metav1.NamespaceNone)
+ if err != nil {
+ return err
+ }
+
+ err = c.Delete(obj.GetName(), &deleteOpts)
+ if err != nil && (errors.IsNotFound(err) || errors.IsConflict(err)) {
+ // We lost a race with something else changing the object
+ log.Debugf("Ignoring error while deleting %s: %s", desc, err)
+ err = nil
+ }
+ if err != nil {
+ return fmt.Errorf("Error deleting %s: %s", desc, err)
+ }
+
+ return nil
+}
+
+func walkObjects(pool dynamic.ClientPool, disco discovery.DiscoveryInterface, listopts metav1.ListOptions, callback func(runtime.Object) error) error {
+ rsrclists, err := disco.ServerResources()
+ if err != nil {
+ return err
+ }
+ for _, rsrclist := range rsrclists {
+ gv, err := schema.ParseGroupVersion(rsrclist.GroupVersion)
+ if err != nil {
+ return err
+ }
+ for _, rsrc := range rsrclist.APIResources {
+ gvk := gv.WithKind(rsrc.Kind)
+
+ if !stringListContains(rsrc.Verbs, "list") {
+ log.Debugf("Don't know how to list %v, skipping", rsrc)
+ continue
+ }
+ client, err := pool.ClientForGroupVersionKind(gvk)
+ if err != nil {
+ return err
+ }
+
+ var ns string
+ if rsrc.Namespaced {
+ ns = metav1.NamespaceAll
+ } else {
+ ns = metav1.NamespaceNone
+ }
+
+ rc := client.Resource(&rsrc, ns)
+ log.Debugf("Listing %s", gvk)
+ obj, err := rc.List(listopts)
+ if err != nil {
+ return err
+ }
+ if err = meta.EachListItem(obj, callback); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func eligibleForGc(obj metav1.Object, gcTag string) bool {
+ for _, ref := range obj.GetOwnerReferences() {
+ if ref.Controller != nil && *ref.Controller {
+ // Has a controller ref
+ return false
+ }
+ }
+
+ a := obj.GetAnnotations()
+
+ strategy, ok := a[AnnotationGcStrategy]
+ if !ok {
+ strategy = GcStrategyAuto
+ }
+
+ return a[AnnotationGcTag] == gcTag &&
+ strategy == GcStrategyAuto
+}
diff --git a/cmd/update_test.go b/cmd/update_test.go
new file mode 100644
index 00000000..70d596a5
--- /dev/null
+++ b/cmd/update_test.go
@@ -0,0 +1,81 @@
+package cmd
+
+import (
+ "testing"
+
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+
+ "github.com/ksonnet/kubecfg/utils"
+)
+
+func TestStringListContains(t *testing.T) {
+ foobar := []string{"foo", "bar"}
+ if stringListContains([]string{}, "") {
+ t.Error("Empty list was not empty")
+ }
+ if !stringListContains(foobar, "foo") {
+ t.Error("Failed to find foo")
+ }
+ if stringListContains(foobar, "baz") {
+ t.Error("Should not contain baz")
+ }
+}
+
+func TestEligibleForGc(t *testing.T) {
+ const myTag = "my-gctag"
+ boolTrue := true
+ o := &unstructured.Unstructured{
+ Object: map[string]interface{}{
+ "apiVersion": "tests/v1alpha1",
+ "kind": "Dummy",
+ },
+ }
+
+ if eligibleForGc(o, myTag) {
+ t.Errorf("%v should not be eligible (no tag)", o)
+ }
+
+ utils.SetMetaDataAnnotation(o, AnnotationGcTag, "unknowntag")
+ if eligibleForGc(o, myTag) {
+ t.Errorf("%v should not be eligible (wrong tag)", o)
+ }
+
+ utils.SetMetaDataAnnotation(o, AnnotationGcTag, myTag)
+ if !eligibleForGc(o, myTag) {
+ t.Errorf("%v should be eligible", o)
+ }
+
+ utils.SetMetaDataAnnotation(o, AnnotationGcStrategy, GcStrategyIgnore)
+ if eligibleForGc(o, myTag) {
+ t.Errorf("%v should not be eligible (strategy=ignore)", o)
+ }
+
+ utils.SetMetaDataAnnotation(o, AnnotationGcStrategy, GcStrategyAuto)
+ if !eligibleForGc(o, myTag) {
+ t.Errorf("%v should be eligible (strategy=auto)", o)
+ }
+
+ // Unstructured.SetOwnerReferences is broken in apimachinery release-1.6
+ // See kubernetes/kubernetes#46817
+ setOwnerRef := func(u *unstructured.Unstructured, ref metav1.OwnerReference) {
+ // This is not a complete nor robust reimplementation
+ c := map[string]interface{}{
+ "kind": ref.Kind,
+ "name": ref.Name,
+ }
+ if ref.Controller != nil {
+ c["controller"] = *ref.Controller
+ }
+ u.Object["metadata"].(map[string]interface{})["ownerReferences"] = []map[string]interface{}{c}
+ }
+ setOwnerRef(o, metav1.OwnerReference{Kind: "foo", Name: "bar"})
+ if !eligibleForGc(o, myTag) {
+ t.Errorf("%v should be eligible (non-controller ownerref)", o)
+ }
+
+ setOwnerRef(o, metav1.OwnerReference{Kind: "foo", Name: "bar", Controller: &boolTrue})
+ if eligibleForGc(o, myTag) {
+ t.Errorf("%v should not be eligible (controller ownerref)", o)
+ }
+}
diff --git a/utils/meta.go b/utils/meta.go
index 9d448a44..8c83afdb 100644
--- a/utils/meta.go
+++ b/utils/meta.go
@@ -4,6 +4,7 @@ import (
"fmt"
"strconv"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/version"
"k8s.io/client-go/discovery"
)
@@ -60,3 +61,13 @@ func (v ServerVersion) Compare(major, minor int) int {
func (v ServerVersion) String() string {
return fmt.Sprintf("%d.%d", v.Major, v.Minor)
}
+
+// SetMetaDataAnnotation sets an annotation value
+func SetMetaDataAnnotation(obj metav1.Object, key, value string) {
+ a := obj.GetAnnotations()
+ if a == nil {
+ a = make(map[string]string)
+ }
+ a[key] = value
+ obj.SetAnnotations(a)
+}
--
GitLab