Split image descriptions (#12)

* Add ability to reference imageDescriptions from pipeline yaml

* Bump version to 0.0.8

cmd/k8s-pipeliner/main.go

@@ -14,7 +14,7 @@ import (
 
 const (
 	// Version defines the current version of k8s-pipeliner
-	Version = "0.0.7"
+	Version = "0.0.8"
 )
 
 func main() {

pipeline/builder/builder.go

@@ -101,7 +101,11 @@ func (b *Builder) buildRunJobStage(index int, s config.Stage) (*types.RunJobStag
 		DNSPolicy:         "ClusterFirst", // hack for now
 	}
 
-	mg, err := ContainersFromManifest(s.RunJob.ManifestFile)
+	parser := &ManifestParser{
+		config: b.pipeline,
+	}
+
+	mg, err := parser.ContainersFromScaffold(s.RunJob)
 	if err != nil {
 		return nil, err
 	}
@@ -130,8 +134,12 @@ func (b *Builder) buildDeployStage(index int, s config.Stage) (*types.DeployStag
 		StageMetadata: buildStageMetadata(s, "deploy", index, b.isLinear),
 	}
 
+	parser := &ManifestParser{
+		config: b.pipeline,
+	}
+
 	for _, group := range s.Deploy.Groups {
-		mg, err := ContainersFromManifest(group.ManifestFile)
+		mg, err := parser.ContainersFromScaffold(group)
 		if err != nil {
 			return nil, err
 		}

pipeline/builder/kubernetes.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/namely/k8s-pipeliner/pipeline/builder/types"
+	"github.com/namely/k8s-pipeliner/pipeline/config"
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -20,33 +21,6 @@ var (
 	ErrUnsupportedManifest = errors.New("builder: manifest type is not supported")
 )
 
-const (
-	// SpinnakerImageDescriptionAccountAnnotation is used for injecting in the docker registry
-	// that should be used when generating the imageDescription struct on a container
-	// field. This should match a docker registry account you've added to spinnaker
-	SpinnakerImageDescriptionAccountAnnotation = "namely.com/spinnaker-image-description-account"
-
-	// SpinnakerImageDescriptionImageIDAnnotation represents the whole repository
-	// Example: registry.namely.com/namely/namely:latest
-	SpinnakerImageDescriptionImageIDAnnotation = "namely.com/spinnaker-image-description-imageid"
-
-	// SpinnakerImageDescriptionRegistryAnnotation is the registry host
-	// Example: registry.namely.com
-	SpinnakerImageDescriptionRegistryAnnotation = "namely.com/spinnaker-image-description-registry"
-
-	// SpinnakerImageDescriptionRepositoryAnnotation is the user / repository name
-	// Example: "namely/namely"
-	SpinnakerImageDescriptionRepositoryAnnotation = "namely.com/spinnaker-image-description-repository"
-
-	// SpinnakerImageDescriptionTagAnnotation is the tag portion of the image ID
-	// Example: "latest"
-	SpinnakerImageDescriptionTagAnnotation = "namely.com/spinnaker-image-description-tag"
-
-	// SpinnakerImageDescriptionOrganizationAnnotation is the registry org that owns the image.
-	// Example: "namely" (where registry.namely.land/namely <- is the org)
-	SpinnakerImageDescriptionOrganizationAnnotation = "namely.com/spinnaker-image-description-organization"
-)
-
 // ManifestGroup keeps a collection of containers from a deployment
 // and metadata associated with them
 type ManifestGroup struct {
@@ -56,13 +30,21 @@ type ManifestGroup struct {
 	VolumeSources []*types.VolumeSource
 }
 
-// ContainersFromManifest loads a kubernetes manifest file and generates
+// ManifestParser handles generating Spinnaker builder types from a kubernetes
+// manifest file (deployments)
+type ManifestParser struct {
+	config *config.Pipeline
+}
+
+// NewManfifestParser initializes and returns a manifest parser for a given pipeline config
+func NewManfifestParser(config *config.Pipeline) *ManifestParser {
+	return &ManifestParser{config}
+}
+
+// ContainersFromGroup loads a kubernetes manifest file and generates
 // spinnaker pipeline containers config from it.
-//
-// NOTE: If your manifest file declares multiple types, only the first will be taken
-// to generate the config.
-func ContainersFromManifest(file string) (*ManifestGroup, error) {
-	f, err := os.Open(file)
+func (mp *ManifestParser) ContainersFromScaffold(scaffold config.ContainerScaffold) (*ManifestGroup, error) {
+	f, err := os.Open(scaffold.Manifest())
 	if err != nil {
 		return nil, err
 	}
@@ -90,10 +72,10 @@ func ContainersFromManifest(file string) (*ManifestGroup, error) {
 
 	switch t := resource.(type) {
 	case *appsv1.Deployment:
-		mg.Containers = deploymentContainers(t)
+		mg.Containers = mp.deploymentContainers(t, scaffold.ImageDescriptionRef())
 		mg.Annotations = t.Annotations
 		mg.Namespace = t.Namespace
-		mg.VolumeSources = volumeSources(t.Spec.Template.Spec.Volumes)
+		mg.VolumeSources = mp.volumeSources(t.Spec.Template.Spec.Volumes)
 	default:
 		return nil, ErrUnsupportedManifest
 	}
@@ -102,7 +84,7 @@ func ContainersFromManifest(file string) (*ManifestGroup, error) {
 }
 
 // converts kubernetes volume sources into builder types
-func volumeSources(vols []corev1.Volume) []*types.VolumeSource {
+func (mp *ManifestParser) volumeSources(vols []corev1.Volume) []*types.VolumeSource {
 	var vs []*types.VolumeSource
 
 	for _, vol := range vols {
@@ -141,20 +123,26 @@ func volumeSources(vols []corev1.Volume) []*types.VolumeSource {
 	return vs
 }
 
-func deploymentContainers(dep *appsv1.Deployment) []*types.Container {
+func (mp *ManifestParser) deploymentContainers(dep *appsv1.Deployment, ref config.ImageDescriptionRef) []*types.Container {
 	var c []*types.Container
 
 	for _, container := range dep.Spec.Template.Spec.Containers {
 		spinContainer := &types.Container{}
 
 		// add the image description first off using the annotations on the container
+		var imageDescription config.ImageDescription
+		for _, desc := range mp.config.ImageDescriptions {
+			if desc.Name == ref.Name && ref.ContainerName == container.Name {
+				imageDescription = desc
+			}
+		}
 		spinContainer.ImageDescription = types.ImageDescription{
-			Account:      dep.Annotations[SpinnakerImageDescriptionAccountAnnotation],
-			ImageID:      dep.Annotations[SpinnakerImageDescriptionImageIDAnnotation],
-			Tag:          dep.Annotations[SpinnakerImageDescriptionTagAnnotation],
-			Repository:   dep.Annotations[SpinnakerImageDescriptionRepositoryAnnotation],
-			Registry:     dep.Annotations[SpinnakerImageDescriptionRegistryAnnotation],
-			Organization: dep.Annotations[SpinnakerImageDescriptionOrganizationAnnotation],
+			Account:      imageDescription.Account,
+			ImageID:      imageDescription.ImageID,
+			Tag:          imageDescription.Tag,
+			Repository:   imageDescription.Repository,
+			Registry:     imageDescription.Registry,
+			Organization: imageDescription.Organization,
 		}
 
 		args := []string{}

pipeline/builder/kubernetes_test.go

@@ -5,17 +5,46 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/namely/k8s-pipeliner/pipeline/builder"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/namely/k8s-pipeliner/pipeline/builder"
+	"github.com/namely/k8s-pipeliner/pipeline/config"
 )
 
+type scaffoldMock struct {
+	manifest            string
+	imageDescriptionRef config.ImageDescriptionRef
+}
+
+func (sm scaffoldMock) Manifest() string {
+	return sm.manifest
+}
+
+func (sm scaffoldMock) ImageDescriptionRef() config.ImageDescriptionRef {
+	return sm.imageDescriptionRef
+}
+
 func TestContainersFromManifests(t *testing.T) {
 	wd, _ := os.Getwd()
 
 	t.Run("Deployment manifests are returned correctly", func(t *testing.T) {
 		file := filepath.Join(wd, "testdata", "deployment.full.yml")
-		group, err := builder.ContainersFromManifest(file)
+		parser := builder.NewManfifestParser(&config.Pipeline{
+			ImageDescriptions: []config.ImageDescription{
+				{
+					Name:    "test-ref",
+					ImageID: "this-is-the-image-id",
+				},
+			},
+		})
+		group, err := parser.ContainersFromScaffold(scaffoldMock{
+			manifest: file,
+			imageDescriptionRef: config.ImageDescriptionRef{
+				Name:          "test-ref",
+				ContainerName: "test-container",
+			},
+		})
 
 		require.NoError(t, err, "error on retrieving the deployment manifests")
 
@@ -31,11 +60,20 @@ func TestContainersFromManifests(t *testing.T) {
 			assert.Equal(t, "/thisisthemount", c.VolumeMounts[0].MountPath)
 			assert.Equal(t, true, c.VolumeMounts[0].ReadOnly)
 		})
+
+		t.Run("Container image descriptions are returned correctly", func(t *testing.T) {
+			c := group.Containers[0]
+
+			assert.Equal(t, "this-is-the-image-id", c.ImageDescription.ImageID)
+		})
 	})
 
 	t.Run("Deployments schemes are converted to latest", func(t *testing.T) {
 		file := filepath.Join(wd, "testdata", "deployment.v1beta1.yml")
-		group, err := builder.ContainersFromManifest(file)
+		parser := builder.NewManfifestParser(&config.Pipeline{})
+		group, err := parser.ContainersFromScaffold(scaffoldMock{
+			manifest: file,
+		})
 
 		require.NoError(t, err, "error on retrieving the deployment manifests")
 
@@ -46,7 +84,10 @@ func TestContainersFromManifests(t *testing.T) {
 
 	t.Run("Volume sources are copied", func(t *testing.T) {
 		file := filepath.Join(wd, "testdata", "deployment.full.yml")
-		group, err := builder.ContainersFromManifest(file)
+		parser := builder.NewManfifestParser(&config.Pipeline{})
+		group, err := parser.ContainersFromScaffold(scaffoldMock{
+			manifest: file,
+		})
 		require.NoError(t, err)
 		require.Len(t, group.VolumeSources, 3)
 

pipeline/builder/testdata/deployment.full.yml

@@ -13,7 +13,8 @@ spec:
         app: example
     spec:
       containers:
-      - command:
+      - name: test-container
+        command:
         - echo
         - hello
         env:

pipeline/config/config.go

@@ -25,10 +25,23 @@ func NewPipeline(r io.Reader) (*Pipeline, error) {
 // Pipeline is the high level struct that contains all of the configuration
 // of a pipeline
 type Pipeline struct {
-	Name        string    `yaml:"name"`
-	Application string    `yaml:"application"`
-	Triggers    []Trigger `yaml:"triggers"`
-	Stages      []Stage   `yaml:"stages"`
+	Name              string             `yaml:"name"`
+	Application       string             `yaml:"application"`
+	Triggers          []Trigger          `yaml:"triggers"`
+	Stages            []Stage            `yaml:"stages"`
+	ImageDescriptions []ImageDescription `yaml:"imageDescriptions"`
+}
+
+// ImageDescription contains the description of an image that can be referenced
+// from stages to inject in an image.
+type ImageDescription struct {
+	Name         string `yaml:"name"`
+	Account      string `yaml:"account"`
+	ImageID      string `yaml:"image_id"`
+	Registry     string `yaml:"registry"`
+	Repository   string `yaml:"repository"`
+	Tag          string `yaml:"tag"`
+	Organization string `yaml:"organization"`
 }
 
 // Trigger contains the fields that are relevant for
@@ -78,20 +91,31 @@ type Container struct {
 
 // RunJobStage is the configuration for a one off job in a spinnaker pipeline
 type RunJobStage struct {
-	ManifestFile string     `yaml:"manifestFile"`
-	Container    *Container `yaml:"container"`
+	ManifestFile     string              `yaml:"manifestFile"`
+	ImageDescription ImageDescriptionRef `yaml:"imageDescription"`
+
+	Container *Container `yaml:"container"`
 }
 
 // DeployStage is the configuration for deploying a cluster of servers (pods)
 type DeployStage struct {
 	Groups []Group `yaml:"groups"`
 }
 
+// ImageDescriptionRef represents a reference to a defined ImageDescription on
+// a given pipeline
+type ImageDescriptionRef struct {
+	Name          string `yaml:"name"`
+	ContainerName string `yaml:"containerName"`
+}
+
 // Group represents a group to be deployed (Think: Kubernetes Pods). Most of the configuration
 // of a group is filled out by the defined manifest file. This means things like commands, env vars,
 // etc, are all pulled into the group spec for you.
 type Group struct {
-	ManifestFile     string   `yaml:"manifestFile"`
+	ManifestFile     string              `yaml:"manifestFile"`
+	ImageDescription ImageDescriptionRef `yaml:"imageDescription"`
+
 	MaxRemainingASGS int      `yaml:"maxRemainingASGS"`
 	ScaleDown        bool     `yaml:"scaleDown"`
 	Stack            string   `yaml:"stack"`
@@ -123,3 +147,25 @@ type ContainerOverrides struct {
 	Args    []string `yaml:"args,omitempty"`
 	Command []string `yaml:"command,omitempty"`
 }
+
+// ContainerScaffold is used to make it easy to get a file and image ref
+// so you can build multiple types of stages (run job or deploys)
+type ContainerScaffold interface {
+	Manifest() string
+	ImageDescriptionRef() ImageDescriptionRef
+}
+
+var _ ContainerScaffold = Group{}
+var _ ContainerScaffold = RunJobStage{}
+
+// Manifest implements ContainerScaffold
+func (g Group) Manifest() string { return g.ManifestFile }
+
+// ImageDescriptionRef implements ContainerScaffold
+func (g Group) ImageDescriptionRef() ImageDescriptionRef { return g.ImageDescription }
+
+// Manifest implements ContainerScaffold
+func (rj RunJobStage) Manifest() string { return rj.ManifestFile }
+
+// ImageDescriptionRef implements ContainerScaffold
+func (rj RunJobStage) ImageDescriptionRef() ImageDescriptionRef { return rj.ImageDescription }

pipeline/config/config_test.go

@@ -29,4 +29,6 @@ func TestNewConfig(t *testing.T) {
 
 	require.Len(t, cfg.Stages[1].Deploy.Groups, 1, "no groups on deploy stage")
 	assert.Equal(t, cfg.Stages[1].Deploy.Groups[0].ManifestFile, "manifests/deploy/connect.yml")
+
+	require.Len(t, cfg.ImageDescriptions, 1, "image descriptions was empty")
 }

pipeline/config/testdata/pipeline.full.yaml

@@ -5,6 +5,14 @@ triggers:
     job: "Connect/job/master"
     master: "namely-jenkins"
     propertyFile: "build.properties"
+imageDescriptions:
+  - name: testImageRef
+    account: account
+    image_id: image_id
+    registry: registry
+    repository: repository
+    tag: tag
+    organization: organization
 stages:
 - account: int-k8s
   name: "Migrate INT"

test-deployment.yml

@@ -3,13 +3,6 @@ kind: Deployment
 metadata:
   name: example
   namespace: production
-  annotations:
-    namely.com/spinnaker-image-description-account: "your-registry"
-    namely.com/spinnaker-image-description-imageid: "${ trigger.properties['docker_image'] }"
-    namely.com/spinnaker-image-description-registry: "your.registry.land"
-    namely.com/spinnaker-image-description-repository: "org/example"
-    namely.com/spinnaker-image-description-tag: "${ trigger.properties['docker_tag'] }"
-    namely.com/spinnaker-load-balancers: "example"
 spec:
   template:
     metadata:
@@ -24,7 +17,8 @@ spec:
               - key: "hello"
                 path: "/my/file/path"
       containers:
-      - command:
+      - name: example
+        command:
         - bundle
         - exec
         - unicorn
@@ -45,7 +39,6 @@ spec:
         # this image doesn't really matter, its going to be replaced, but its not to show intent
         image: your.registry.land/org/example:latest
         imagePullPolicy: IfNotPresent
-        name: example
         ports:
         - containerPort: 80
           name: http