Tests for Kubernetes manifests deserialization (#1)

* Forgot to do dynamic organizations

* Add Organization to pipeline example

* Add tests for converting k8s manifests to internal types

* Remove dead files from spinnaker

* Add Makefile for pushing releases for k8s-pipeliner

* Bubble up scheme conversion errors

.gitignore

@@ -0,0 +1 @@
+bin/

Makefile

@@ -0,0 +1,33 @@
+# assign the current version from the binary
+VERSION = $(shell go run cmd/k8s-pipeliner/main.go --version | awk '{print $$3}')
+
+test:
+	go test -cover ./...
+	golint -set_exit_status ./...
+
+build:
+	mkdir -p bin/darwin
+	mkdir -p bin/linux
+	GOOS=darwin go build -o bin/darwin/k8s-pipeliner cmd/k8s-pipeliner/main.go
+	GOOS=linux go build -o bin/linux/k8s-pipeliner cmd/k8s-pipeliner/main.go
+
+release: test build;
+	git tag v$(VERSION) && git push --tags
+	github-release release \
+    --user namely \
+    --repo k8s-pipeliner \
+    --tag v$(VERSION) \
+    --name "k8s-pipeliner release $(VERSION)" \
+    --description "";
+	github-release upload \
+		--user namely \
+		--repo k8s-pipeliner \
+		--tag v$(VERSION) \
+		--name "k8s-pipeliner-osx-amd64" \
+		--file bin/darwin/k8s-pipeliner;
+	github-release upload \
+		--user namely \
+		--repo k8s-pipeliner \
+		--tag v$(VERSION) \
+		--name "k8s-pipeliner-linux-amd64" \
+		--file bin/linux/k8s-pipeliner;

README.md

@@ -86,6 +86,7 @@ metadata:
     namely.com/spinnaker-image-description-imageid: "${ trigger.properties['docker_image'] }"
     namely.com/spinnaker-image-description-registry: "your.registry.land"
     namely.com/spinnaker-image-description-repository: "org/example"
+    namely.com/spinnaker-image-description-organization: "namely"
     namely.com/spinnaker-image-description-tag: "${ trigger.properties['docker_tag'] }"
     namely.com/spinnaker-load-balancers: "example"
 ```

cmd/k8s-pipeliner/main.go

@@ -11,11 +11,17 @@ import (
 	"github.com/urfave/cli"
 )
 
+const (
+	// Version defines the current version of k8s-pipeliner
+	Version = "0.0.1"
+)
+
 func main() {
 	app := cli.NewApp()
 	app.Name = "k8s-pipeliner"
 	app.Description = "create spinnaker pipelines from kubernetes clusters"
 	app.Flags = []cli.Flag{}
+	app.Version = Version
 
 	app.Commands = []cli.Command{
 		{

pipeline/builder/kubernetes.go

@@ -2,15 +2,15 @@ package builder
 
 import (
 	"errors"
+	"io/ioutil"
 	"os"
 	"strings"
 
 	"github.com/namely/k8s-pipeliner/pipeline/builder/types"
 
-	"k8s.io/api/apps/v1beta2"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	appsv1 "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/client-go/kubernetes/scheme"
 )
 
 var (
@@ -41,6 +41,10 @@ const (
 	// Example: "latest"
 	SpinnakerImageDescriptionTagAnnotation = "namely.com/spinnaker-image-description-tag"
 
+	// SpinnakerImageDescriptionOrganizationAnnotation is the registry org that owns the image.
+	// Example: "namely" (where registry.namely.land/namely <- is the org)
+	SpinnakerImageDescriptionOrganizationAnnotation = "namely.com/spinnaker-image-description-organization"
+
 	// SpinnakerLoadBalancersAnnotations is a comma separated list of load balancers
 	// defined in Spinnaker that should be attached to a cluster
 	// Example: "catalog,catalog-public"
@@ -66,44 +70,40 @@ func ContainersFromManifest(file string) (*ManifestGroup, error) {
 		return nil, err
 	}
 
-	d := yaml.NewYAMLOrJSONDecoder(f, 4096)
-
-	ext := runtime.RawExtension{}
-	if dErr := d.Decode(&ext); dErr != nil {
-		return nil, dErr
-	}
-
-	versions := &runtime.VersionedObjects{}
-	_, gvk, err := unstructured.UnstructuredJSONScheme.Decode(ext.Raw, nil, versions)
+	b, err := ioutil.ReadAll(f)
 	if err != nil {
 		return nil, err
 	}
 
-	// seek back to the beginning of the file so we can unmarshal into the correct
-	// type once we've determined it
-	if _, err := f.Seek(0, 0); err != nil {
+	decode := scheme.Codecs.UniversalDeserializer().Decode
+	obj, g, err := decode(b, nil, nil)
+	if err != nil {
 		return nil, err
 	}
+
 	var mg ManifestGroup
+	var resource runtime.Object
 
-	switch gvk.Kind {
-	case "Deployment":
-		dep := &v1beta2.Deployment{}
-		if err := d.Decode(dep); err != nil {
+	if g.Kind == "Deployment" {
+		resource = &appsv1.Deployment{}
+		if err := scheme.Scheme.Convert(obj, resource, nil); err != nil {
 			return nil, err
 		}
+	}
 
-		mg.Containers = deploymentContainers(dep)
-		mg.Annotations = dep.Annotations
-		mg.Namespace = dep.Namespace
+	switch t := resource.(type) {
+	case *appsv1.Deployment:
+		mg.Containers = deploymentContainers(t)
+		mg.Annotations = t.Annotations
+		mg.Namespace = t.Namespace
 	default:
 		return nil, ErrUnsupportedManifest
 	}
 
 	return &mg, nil
 }
 
-func deploymentContainers(dep *v1beta2.Deployment) []*types.Container {
+func deploymentContainers(dep *appsv1.Deployment) []*types.Container {
 	var c []*types.Container
 	for _, container := range dep.Spec.Template.Spec.Containers {
 		spinContainer := &types.Container{}
@@ -115,7 +115,7 @@ func deploymentContainers(dep *v1beta2.Deployment) []*types.Container {
 			Tag:          dep.Annotations[SpinnakerImageDescriptionTagAnnotation],
 			Repository:   dep.Annotations[SpinnakerImageDescriptionRepositoryAnnotation],
 			Registry:     dep.Annotations[SpinnakerImageDescriptionRegistryAnnotation],
-			Organization: "namely",
+			Organization: dep.Annotations[SpinnakerImageDescriptionOrganizationAnnotation],
 		}
 
 		args := []string{}

pipeline/builder/kubernetes_test.go

@@ -0,0 +1,37 @@
+package builder_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/namely/k8s-pipeliner/pipeline/builder"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestContainersFromManifests(t *testing.T) {
+	wd, _ := os.Getwd()
+
+	t.Run("Deployment manifests are returned correctly", func(t *testing.T) {
+		file := filepath.Join(wd, "testdata", "deployment.full.yml")
+		group, err := builder.ContainersFromManifest(file)
+
+		require.NoError(t, err, "error on retrieving the deployment manifests")
+
+		assert.Len(t, group.Containers, 1)
+		assert.Len(t, group.Annotations, 2)
+		assert.Equal(t, "fake-namespace", group.Namespace)
+	})
+
+	t.Run("Deployments schemes are converted to latest", func(t *testing.T) {
+		file := filepath.Join(wd, "testdata", "deployment.v1beta1.yml")
+		group, err := builder.ContainersFromManifest(file)
+
+		require.NoError(t, err, "error on retrieving the deployment manifests")
+
+		assert.Len(t, group.Containers, 1)
+		assert.Len(t, group.Annotations, 2)
+		assert.Equal(t, "fake-namespace", group.Namespace)
+	})
+}

pipeline/builder/testdata/deployment.full.yml

@@ -0,0 +1,22 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: example
+  namespace: fake-namespace
+  annotations:
+    fake-annotation-1: "Hello"
+    fake-annotation-2: "World"
+spec:
+  template:
+    metadata:
+      labels:
+        app: example
+    spec:
+      containers:
+      - command:
+        - echo
+        - hello
+        env:
+        - name: WHATS_THE_WORD
+          value: "bird is the word"
+        image: bird.word/latest

pipeline/builder/testdata/deployment.v1beta1.yml

@@ -0,0 +1,22 @@
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: example
+  namespace: fake-namespace
+  annotations:
+    fake-annotation-1: "Hello"
+    fake-annotation-2: "World"
+spec:
+  template:
+    metadata:
+      labels:
+        app: example
+    spec:
+      containers:
+      - command:
+        - echo
+        - hello
+        env:
+        - name: WHATS_THE_WORD
+          value: "bird is the word"
+        image: bird.word/latest