Add additional k8s container fields (#16)

* Add envFrom as a reference

* Add readiness and liveness probes to Spinnaker JSON

* Bump version to 0.0.11

* Add quick test for probe conversion

cmd/k8s-pipeliner/main.go

@@ -14,7 +14,7 @@ import (
 
 const (
 	// Version defines the current version of k8s-pipeliner
-	Version = "0.0.10"
+	Version = "0.0.11"
 )
 
 func main() {

pipeline/builder/kubernetes.go

@@ -202,6 +202,27 @@ func (mp *ManifestParser) deploymentContainers(dep *appsv1.Deployment, scaffold
 			spinContainer.EnvVars = append(spinContainer.EnvVars, e)
 		}
 
+		for _, envFrom := range container.EnvFrom {
+			var e types.EnvFromSource
+			e.Prefix = envFrom.Prefix
+
+			if cmRef := envFrom.ConfigMapRef; cmRef != nil {
+				e.ConfigMapSource = &types.EnvFromConfigMapSource{
+					Name: cmRef.Name,
+				}
+			}
+			// TODO(bobbytables): Add secretRefs as well for envFrom sources
+			spinContainer.EnvFrom = append(spinContainer.EnvFrom, e)
+		}
+
+		if probe := container.LivenessProbe; probe != nil {
+			spinContainer.LivenessProbe = spinnakerProbeHandler(probe)
+		}
+
+		if probe := container.ReadinessProbe; probe != nil {
+			spinContainer.ReadinessProbe = spinnakerProbeHandler(probe)
+		}
+
 		// add all of the volume mounts
 		for _, vm := range container.VolumeMounts {
 			spinContainer.VolumeMounts = append(spinContainer.VolumeMounts, types.VolumeMount{
@@ -216,3 +237,46 @@ func (mp *ManifestParser) deploymentContainers(dep *appsv1.Deployment, scaffold
 
 	return c
 }
+
+func spinnakerProbeHandler(probe *corev1.Probe) *types.Probe {
+	h := types.ProbeHandler{}
+
+	if httpGet := probe.HTTPGet; httpGet != nil {
+		h.Type = "HTTP"
+		h.HTTPGetAction = &types.HTTPGetAction{
+			Path:      httpGet.Path,
+			Port:      httpGet.Port.IntValue(),
+			URIScheme: string(httpGet.Scheme),
+		}
+
+		for _, header := range httpGet.HTTPHeaders {
+			h.HTTPGetAction.HTTPHeaders = append(h.HTTPGetAction.HTTPHeaders, types.HTTPGetActionHeaders{
+				Name:  header.Name,
+				Value: header.Value,
+			})
+		}
+	}
+
+	if exec := probe.Exec; exec != nil {
+		h.ExecAction = &types.ExecAction{
+			Commands: exec.Command,
+		}
+		h.Type = "EXEC"
+	}
+
+	if tcp := probe.TCPSocket; tcp != nil {
+		h.TCPSocketAction = &types.TCPSocketAction{
+			Port: tcp.Port.IntValue(),
+		}
+		h.Type = "TCP"
+	}
+
+	return &types.Probe{
+		FailureThreshold:    probe.FailureThreshold,
+		InitialDelaySeconds: probe.InitialDelaySeconds,
+		PeriodSeconds:       probe.PeriodSeconds,
+		SuccessThreshold:    probe.SuccessThreshold,
+		TimeoutSeconds:      probe.TimeoutSeconds,
+		Handler:             h,
+	}
+}

pipeline/builder/kubernetes_test.go

@@ -121,4 +121,33 @@ func TestContainersFromManifests(t *testing.T) {
 			assert.Equal(t, ed.Type, "EMPTYDIR")
 		})
 	})
+
+	t.Run("EnvFrom sources are copied in", func(t *testing.T) {
+		file := filepath.Join(wd, "testdata", "deployment.envfrom.yml")
+		parser := builder.NewManfifestParser(&config.Pipeline{})
+		group, err := parser.ContainersFromScaffold(scaffoldMock{
+			manifest: file,
+		})
+		require.NoError(t, err)
+
+		container := group.Containers[0]
+		require.Len(t, container.EnvFrom, 1)
+		require.NotNil(t, container.EnvFrom[0].ConfigMapSource)
+		assert.Equal(t, "dummy-ref", container.EnvFrom[0].ConfigMapSource.Name)
+		assert.Equal(t, "some-prefix", container.EnvFrom[0].Prefix)
+	})
+
+	t.Run("LivenessProbe is copied in the correct format", func(t *testing.T) {
+		file := filepath.Join(wd, "testdata", "deployment.probes.yml")
+		parser := builder.NewManfifestParser(&config.Pipeline{})
+		group, err := parser.ContainersFromScaffold(scaffoldMock{
+			manifest: file,
+		})
+		require.NoError(t, err)
+
+		container := group.Containers[0]
+
+		require.NotNil(t, container.LivenessProbe)
+		require.NotNil(t, container.ReadinessProbe)
+	})
 }

pipeline/builder/testdata/deployment.envfrom.yml

@@ -0,0 +1,25 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: example
+  namespace: fake-namespace
+  annotations:
+    fake-annotation-1: "Hello"
+    fake-annotation-2: "World"
+spec:
+  template:
+    metadata:
+      labels:
+        app: example
+      annotations:
+        test: "annotations"
+    spec:
+      containers:
+      - name: test-container
+        command:
+        - echo
+        - hello
+        envFrom:
+        - configMapRef:
+            name: dummy-ref
+          prefix: some-prefix

pipeline/builder/testdata/deployment.probes.yml

@@ -0,0 +1,27 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: example
+  namespace: fake-namespace
+  annotations:
+    fake-annotation-1: "Hello"
+    fake-annotation-2: "World"
+spec:
+  template:
+    metadata:
+      labels:
+        app: example
+      annotations:
+        test: "annotations"
+    spec:
+      containers:
+      - name: test-container
+        command:
+        - echo
+        - hello
+        livenessProbe:
+          exec:
+            command: ["say", "hello"]
+        readinessProbe:
+          exec:
+            command: ["say", "hello"]

pipeline/builder/types/types.go

@@ -18,7 +18,7 @@ type Trigger interface {
 
 // StageMetadata is the common components of a stage in spinnaker such as name
 type StageMetadata struct {
-	RefID                string         `json:"refId"`
+	RefID                string         `json:"refId,omitempty"`
 	RequisiteStageRefIds []string       `json:"requisiteStageRefIds,omitempty"`
 	Name                 string         `json:"name"`
 	Type                 string         `json:"type"`
@@ -56,11 +56,11 @@ type RunJobStage struct {
 	Application       string            `json:"application"`
 	CloudProvider     string            `json:"cloudProvider"`
 	CloudProviderType string            `json:"cloudProviderType"`
-	Container         *Container        `json:"container"`
+	Container         *Container        `json:"container,omitempty"`
 	DNSPolicy         string            `json:"dnsPolicy"`
-	Labels            map[string]string `json:"labels"`
+	Labels            map[string]string `json:"labels,omitempty"`
 	Namespace         string            `json:"namespace"`
-	VolumeSources     []interface{}     `json:"volumeSources"`
+	VolumeSources     []interface{}     `json:"volumeSources,omitempty"`
 }
 
 func (rjs RunJobStage) spinnakerStage() {}
@@ -103,32 +103,33 @@ type Cluster struct {
 	InterestingHealthProviderNames []string          `json:"interestingHealthProviderNames"`
 	LoadBalancers                  []string          `json:"loadBalancers"`
 	MaxRemainingAsgs               int               `json:"maxRemainingAsgs"`
-	NodeSelector                   map[string]string `json:"nodeSelector"`
-	PodAnnotations                 map[string]string `json:"podAnnotations"`
+	NodeSelector                   map[string]string `json:"nodeSelector,omitempty"`
+	PodAnnotations                 map[string]string `json:"podAnnotations,omitempty"`
 	Provider                       string            `json:"provider"`
 
 	// Region is just a kubernetes namespace
 	Region    string `json:"region"`
 	Namespace string `json:"namespace"`
 
-	ReplicaSetAnnotations         map[string]string `json:"replicaSetAnnotations"`
+	ReplicaSetAnnotations         map[string]string `json:"replicaSetAnnotations,omitempty"`
 	ScaleDown                     bool              `json:"scaleDown"`
-	SecurityGroups                []interface{}     `json:"securityGroups"`
+	SecurityGroups                []interface{}     `json:"securityGroups,omitempty"`
 	Stack                         string            `json:"stack"`
 	Details                       string            `json:"freeFormDetails"`
 	Strategy                      string            `json:"strategy"`
 	TargetSize                    int               `json:"targetSize"`
 	TerminationGracePeriodSeconds int               `json:"terminationGracePeriodSeconds"`
-	VolumeSources                 []*VolumeSource   `json:"volumeSources"`
+	VolumeSources                 []*VolumeSource   `json:"volumeSources,omitempty"`
 	DelayBeforeDisableSec         int               `json:"delayBeforeDisableSec,omitempty"`
 }
 
 // Container is a representation of a container to be deployed either as a job
 // or within a cluster
 type Container struct {
-	Args             []string         `json:"args"`
-	Command          []string         `json:"command"`
-	EnvVars          []EnvVar         `json:"envVars"`
+	Args             []string         `json:"args,omitempty"`
+	Command          []string         `json:"command,omitempty"`
+	EnvVars          []EnvVar         `json:"envVars,omitempty"`
+	EnvFrom          []EnvFromSource  `json:"envFrom,omitempty"`
 	ImageDescription ImageDescription `json:"imageDescription"`
 	ImagePullPolicy  string           `json:"imagePullPolicy"`
 	Limits           Resources        `json:"limits"`
@@ -138,6 +139,22 @@ type Container struct {
 	Ports []Port `json:"ports"`
 
 	VolumeMounts []VolumeMount `json:"volumeMounts"`
+
+	LivenessProbe  *Probe `json:"livenessProbe"`
+	ReadinessProbe *Probe `json:"readinessProbe"`
+}
+
+// EnvFromSource is used to pull in a config map as a list of environment
+// variables
+type EnvFromSource struct {
+	Prefix          string                  `json:"prefix"`
+	ConfigMapSource *EnvFromConfigMapSource `json:"configMapRef"`
+}
+
+// EnvFromConfigMapSource is used to pull in a configmap for key/value envVars
+type EnvFromConfigMapSource struct {
+	Name     string `json:"name"`
+	Optional bool   `json:"optional"`
 }
 
 // VolumeMount describes a mount that should be mounted in to the container
@@ -242,3 +259,45 @@ type SecretVolumeSource struct {
 	SecretName string             `json:"secretName"`
 	Items      []corev1.KeyToPath `json:"items"`
 }
+
+// Probe is a probe against a container for things such as liveness or readiness
+type Probe struct {
+	FailureThreshold    int32        `json:"failureThreshold"`
+	InitialDelaySeconds int32        `json:"initialDelaySeconds"`
+	PeriodSeconds       int32        `json:"periodSeconds"`
+	SuccessThreshold    int32        `json:"successThreshold"`
+	TimeoutSeconds      int32        `json:"timeoutSeconds"`
+	Handler             ProbeHandler `json:"handler"`
+}
+
+// ProbeHandler represents all of the different types of probes
+type ProbeHandler struct {
+	ExecAction      *ExecAction      `json:"execAction,omitempty"`
+	HTTPGetAction   *HTTPGetAction   `json:"httpGetAction,omitempty"`
+	TCPSocketAction *TCPSocketAction `json:"tcpSocketAction,omitempty"`
+	Type            string           `json:"type"`
+}
+
+// ExecAction is a probe type that runs a command
+type ExecAction struct {
+	Commands []string `json:"commands"`
+}
+
+// HTTPGetAction a probe type that hits an HTTP endpoint in the container
+type HTTPGetAction struct {
+	HTTPHeaders []HTTPGetActionHeaders `json:"httpHeaders"`
+	Path        string                 `json:"path"`
+	Port        int                    `json:"port"`
+	URIScheme   string                 `json:"uriScheme"`
+}
+
+// HTTPGetActionHeaders is a key/value struct for headers used in a HTTP probe
+type HTTPGetActionHeaders struct {
+	Name  string `json:"name"`
+	Value string `json:"value"`
+}
+
+// TCPSocketAction checks if a TCP connection can be opened against the given port
+type TCPSocketAction struct {
+	Port int `json:"port"`
+}

test-deployment.yml

@@ -20,6 +20,11 @@ spec:
                 path: "/my/file/path"
       containers:
       - name: example
+        livenessProbe:
+          exec:
+            command:
+            - cat
+            - /tmp/healthy
         command:
         - bundle
         - exec