merge master

Makefile

@@ -97,6 +97,7 @@ mocks:
 	mockgen -source ./pkg/apis/batch/syncer.go -destination ./pkg/apis/batch/syncer_mock.go -package batch
 	mockgen -source ./pkg/apis/batch/syncerfactory.go -destination ./pkg/apis/batch/syncerfactory_mock.go -package batch
 	mockgen -source ./radix-operator/common/handler.go -destination ./radix-operator/common/handler_mock.go -package common
+	mockgen -source ./pipeline-runner/wait/job.go -destination ./pipeline-runner/wait/job_mock.go -package wait
 
 .PHONY: build-pipeline
 build-pipeline:

pipeline-runner/utils/tekton.go → pipeline-runner/internal/tekton.go

@@ -1,4 +1,4 @@
-package utils
+package internal
 
 import (
 	"fmt"

pipeline-runner/main.go

@@ -9,7 +9,7 @@ import (
 	pipe "github.com/equinor/radix-operator/pipeline-runner/pipelines"
 	"github.com/equinor/radix-operator/pkg/apis/defaults"
 	"github.com/equinor/radix-operator/pkg/apis/pipeline"
-	"github.com/equinor/radix-operator/pkg/apis/radix/v1"
+	v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1"
 	"github.com/equinor/radix-operator/pkg/apis/utils"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -71,7 +71,7 @@ func prepareRunner(pipelineArgs *model.PipelineArguments) (*pipe.PipelineRunner,
 		return nil, err
 	}
 
-	pipelineRunner := pipe.InitRunner(client, radixClient, prometheusOperatorClient, secretProviderClient, pipelineDefinition, pipelineArgs.AppName)
+	pipelineRunner := pipe.NewRunner(client, radixClient, prometheusOperatorClient, secretProviderClient, pipelineDefinition, pipelineArgs.AppName)
 
 	err = pipelineRunner.PrepareRun(pipelineArgs)
 	if err != nil {

pipeline-runner/model/pipelineInfo.go

@@ -2,11 +2,13 @@ package model
 
 import (
 	"fmt"
-	"github.com/equinor/radix-operator/pkg/apis/defaults"
-	"github.com/equinor/radix-operator/pkg/apis/utils/conditions"
+	"path/filepath"
 	"strings"
 	"time"
 
+	"github.com/equinor/radix-operator/pkg/apis/defaults"
+	"github.com/equinor/radix-operator/pkg/apis/utils/conditions"
+
 	"github.com/equinor/radix-common/utils/maps"
 	application "github.com/equinor/radix-operator/pkg/apis/applicationconfig"
 	"github.com/equinor/radix-operator/pkg/apis/pipeline"
@@ -47,6 +49,7 @@ type PipelineInfo struct {
 
 	// Holds information on the images referred to by their respective components
 	ComponentImages map[string]pipeline.ComponentImage
+
 	// Prepare pipeline job build context
 	PrepareBuildContext *PrepareBuildContext
 	StopPipeline        bool
@@ -351,10 +354,6 @@ func getDockerfileName(name string) string {
 }
 
 func getContext(sourceFolder string) string {
-	sourceFolder = strings.Trim(sourceFolder, ".")
-	sourceFolder = strings.Trim(sourceFolder, "/")
-	if sourceFolder == "" {
-		return fmt.Sprintf("%s/", git.Workspace)
-	}
-	return fmt.Sprintf("%s/%s/", git.Workspace, sourceFolder)
+	sourceRoot := filepath.Join("/", sourceFolder)
+	return fmt.Sprintf("%s/", filepath.Join(git.Workspace, sourceRoot))
 }

pipeline-runner/pipelines/app.go

@@ -1,4 +1,4 @@
-package onpush
+package pipelines
 
 import (
 	"context"
@@ -32,9 +32,8 @@ type PipelineRunner struct {
 	pipelineInfo             *model.PipelineInfo
 }
 
-// InitRunner constructor
-func InitRunner(kubeclient kubernetes.Interface, radixclient radixclient.Interface, prometheusOperatorClient monitoring.Interface, secretsstorevclient secretsstorevclient.Interface, definition *pipeline.Definition, appName string) PipelineRunner {
-
+// NewRunner constructor
+func NewRunner(kubeclient kubernetes.Interface, radixclient radixclient.Interface, prometheusOperatorClient monitoring.Interface, secretsstorevclient secretsstorevclient.Interface, definition *pipeline.Definition, appName string) PipelineRunner {
 	kubeUtil, _ := kube.New(kubeclient, radixclient, secretsstorevclient)
 	handler := PipelineRunner{
 		definition:               definition,
@@ -106,10 +105,10 @@ func (cli *PipelineRunner) TearDown() {
 
 func (cli *PipelineRunner) initStepImplementations(registration *v1.RadixRegistration) []model.Step {
 	stepImplementations := make([]model.Step, 0)
-	stepImplementations = append(stepImplementations, steps.NewPreparePipelinesStep())
+	stepImplementations = append(stepImplementations, steps.NewPreparePipelinesStep(nil))
 	stepImplementations = append(stepImplementations, steps.NewApplyConfigStep())
-	stepImplementations = append(stepImplementations, steps.NewBuildStep())
-	stepImplementations = append(stepImplementations, steps.NewRunPipelinesStep())
+	stepImplementations = append(stepImplementations, steps.NewBuildStep(nil))
+	stepImplementations = append(stepImplementations, steps.NewRunPipelinesStep(nil))
 	stepImplementations = append(stepImplementations, steps.NewDeployStep(kube.NewNamespaceWatcherImpl(cli.kubeclient)))
 	stepImplementations = append(stepImplementations, steps.NewPromoteStep())
 

pipeline-runner/pipelines/app_test.go

@@ -1,9 +1,10 @@
-package onpush
+package pipelines_test
 
 import (
 	"testing"
 
 	"github.com/equinor/radix-operator/pipeline-runner/model"
+	"github.com/equinor/radix-operator/pipeline-runner/pipelines"
 	"github.com/equinor/radix-operator/pkg/apis/pipeline"
 	v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1"
 	commonTest "github.com/equinor/radix-operator/pkg/apis/test"
@@ -27,7 +28,7 @@ func setupTest(t *testing.T) (*kubernetes.Clientset, *radix.Clientset, *secretpr
 func TestPrepare_NoRegistration_NotValid(t *testing.T) {
 	kubeclient, radixclient, secretproviderclient, _ := setupTest(t)
 	pipelineDefinition, _ := pipeline.GetPipelineFromName(string(v1.BuildDeploy))
-	cli := InitRunner(kubeclient, radixclient, &monitoring.Clientset{}, secretproviderclient, pipelineDefinition, "any-app")
+	cli := pipelines.NewRunner(kubeclient, radixclient, &monitoring.Clientset{}, secretproviderclient, pipelineDefinition, "any-app")
 
 	err := cli.PrepareRun(&model.PipelineArguments{})
 	assert.Error(t, err)

pipeline-runner/steps/apply_radixconfig.go

@@ -50,13 +50,20 @@ func (cli *ApplyConfigStepImplementation) ErrorMsg(err error) string {
 
 // Run Override of default step method
 func (cli *ApplyConfigStepImplementation) Run(pipelineInfo *model.PipelineInfo) error {
-	// Get radix application from config map
+	// Get pipeline info from configmap created by prepare pipeline step
 	namespace := utils.GetAppNamespace(cli.GetAppName())
 	configMap, err := cli.GetKubeutil().GetConfigMap(namespace, pipelineInfo.RadixConfigMapName)
 	if err != nil {
 		return err
 	}
 
+	// Read build context info from configmap
+	pipelineInfo.PrepareBuildContext, err = getPrepareBuildContextContent(configMap)
+	if err != nil {
+		return err
+	}
+
+	// Read radixconfig from configmap
 	configFileContent, ok := configMap.Data[pipelineDefaults.PipelineConfigMapContent]
 	if !ok {
 		return fmt.Errorf("failed load RadixApplication from ConfigMap")
@@ -81,11 +88,6 @@ func (cli *ApplyConfigStepImplementation) Run(pipelineInfo *model.PipelineInfo)
 	// Set back to pipeline
 	pipelineInfo.SetApplicationConfig(applicationConfig)
 
-	pipelineInfo.PrepareBuildContext, err = getPrepareBuildContextContent(configMap)
-	if err != nil {
-		return err
-	}
-
 	if pipelineInfo.PipelineArguments.PipelineType == string(v1.BuildDeploy) {
 		gitCommitHash, gitTags := cli.getHashAndTags(namespace, pipelineInfo)
 		err = validate.GitTagsContainIllegalChars(gitTags)

pipeline-runner/steps/build.go

@@ -5,23 +5,41 @@ import (
 	"fmt"
 
 	"github.com/equinor/radix-operator/pipeline-runner/model"
+	pipelinewait "github.com/equinor/radix-operator/pipeline-runner/wait"
 	jobUtil "github.com/equinor/radix-operator/pkg/apis/job"
+	"github.com/equinor/radix-operator/pkg/apis/kube"
 	"github.com/equinor/radix-operator/pkg/apis/pipeline"
+	v1 "github.com/equinor/radix-operator/pkg/apis/radix/v1"
 	"github.com/equinor/radix-operator/pkg/apis/utils"
+	radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned"
+	monitoring "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned"
 	log "github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
 )
 
 // BuildStepImplementation Step to build docker image
 type BuildStepImplementation struct {
 	stepType pipeline.StepType
 	model.DefaultStepImplementation
+	jobWaiter pipelinewait.JobCompletionWaiter
 }
 
-// NewBuildStep Constructor
-func NewBuildStep() model.Step {
-	return &BuildStepImplementation{
-		stepType: pipeline.BuildStep,
+// NewBuildStep Constructor.
+// jobWaiter is optional and will be set by Init(...) function if nil.
+func NewBuildStep(jobWaiter pipelinewait.JobCompletionWaiter) model.Step {
+	step := &BuildStepImplementation{
+		stepType:  pipeline.BuildStep,
+		jobWaiter: jobWaiter,
+	}
+
+	return step
+}
+
+func (step *BuildStepImplementation) Init(kubeclient kubernetes.Interface, radixclient radixclient.Interface, kubeutil *kube.Kube, prometheusOperatorClient monitoring.Interface, rr *v1.RadixRegistration) {
+	step.DefaultStepImplementation.Init(kubeclient, radixclient, kubeutil, prometheusOperatorClient, rr)
+	if step.jobWaiter == nil {
+		step.jobWaiter = pipelinewait.NewJobCompletionWaiter(kubeclient)
 	}
 }
 
@@ -84,7 +102,7 @@ func (cli *BuildStepImplementation) Run(pipelineInfo *model.PipelineInfo) error
 		return err
 	}
 
-	return cli.GetKubeutil().WaitForCompletionOf(job)
+	return cli.jobWaiter.Wait(job)
 }
 
 func needToBuildComponents(componentImages map[string]pipeline.ComponentImage) bool {

pipeline-runner/steps/build_acr.go

@@ -109,18 +109,19 @@ func getACRBuildJobVolumes(defaultMode *int32, buildSecrets []corev1.EnvVar) []c
 			},
 		},
 	}
-	if len(buildSecrets) == 0 {
-		return volumes
-	}
-	volumes = append(volumes,
-		corev1.Volume{
-			Name: defaults.BuildSecretsName,
-			VolumeSource: corev1.VolumeSource{
-				Secret: &corev1.SecretVolumeSource{
-					SecretName: defaults.BuildSecretsName,
+
+	if len(buildSecrets) > 0 {
+		volumes = append(volumes,
+			corev1.Volume{
+				Name: defaults.BuildSecretsName,
+				VolumeSource: corev1.VolumeSource{
+					Secret: &corev1.SecretVolumeSource{
+						SecretName: defaults.BuildSecretsName,
+					},
 				},
-			},
-		})
+			})
+	}
+
 	return volumes
 }
 
@@ -278,7 +279,7 @@ func createACRBuildContainers(appName string, pipelineInfo *model.PipelineInfo,
 		if isUsingBuildKit(pipelineInfo) {
 			containerCommand = getBuildahContainerCommand(pipelineInfo, containerRegistry, secretMountsArgsString,
 				componentImage.Context, componentImage.Dockerfile, componentImage.ImagePath,
-				clusterTypeImage, clusterNameImage)
+				clusterTypeImage, clusterNameImage, pushImage)
 			container.Command = containerCommand
 			container.Resources.Requests = map[corev1.ResourceName]resource.Quantity{
 				corev1.ResourceCPU:    resource.MustParse(pipelineInfo.PipelineArguments.Builder.ResourcesRequestsCPU),
@@ -318,26 +319,28 @@ func getBuildAcrJobContainerVolumeMounts(azureServicePrincipleContext string, bu
 	return volumeMounts
 }
 
-func getBuildahContainerCommand(pipelineInfo *model.PipelineInfo, containerImageRegistry, secretArgsString, context, dockerFileName, imageTag, clusterTypeImageTag, clusterNameImageTag string) []string {
-	return []string{
-		"/bin/bash",
-		"-c",
-		fmt.Sprintf("/usr/bin/buildah login --username ${BUILDAH_USERNAME} --password ${BUILDAH_PASSWORD} %s && "+
-			"/usr/bin/buildah build --storage-driver=vfs --isolation=chroot "+
-			"--jobs 0 %s --file %s%s "+
-			"--build-arg RADIX_GIT_COMMIT_HASH=\"${RADIX_GIT_COMMIT_HASH}\" "+
-			"--build-arg RADIX_GIT_TAGS=\"${RADIX_GIT_TAGS}\" "+
-			"--build-arg BRANCH=\"${BRANCH}\" "+
-			"--build-arg TARGET_ENVIRONMENTS=\"${TARGET_ENVIRONMENTS}\" "+
-			"--tag %s --tag %s --tag %s %s && "+
+func getBuildahContainerCommand(pipelineInfo *model.PipelineInfo, containerImageRegistry, secretArgsString, context, dockerFileName, imageTag, clusterTypeImageTag, clusterNameImageTag string, pushImage bool) []string {
+	cmd := fmt.Sprintf("/usr/bin/buildah login --username ${BUILDAH_USERNAME} --password ${BUILDAH_PASSWORD} %s && "+
+		"/usr/bin/buildah build --storage-driver=vfs --isolation=chroot "+
+		"--jobs 0 %s --file %s%s "+
+		"--build-arg RADIX_GIT_COMMIT_HASH=\"${RADIX_GIT_COMMIT_HASH}\" "+
+		"--build-arg RADIX_GIT_TAGS=\"${RADIX_GIT_TAGS}\" "+
+		"--build-arg BRANCH=\"${BRANCH}\" "+
+		"--build-arg TARGET_ENVIRONMENTS=\"${TARGET_ENVIRONMENTS}\" "+
+		"--tag %s --tag %s --tag %s %s",
+		containerImageRegistry, secretArgsString, context, dockerFileName,
+		imageTag, clusterTypeImageTag, clusterNameImageTag,
+		context)
+
+	if pushImage {
+		cmd = fmt.Sprintf("%s && "+
 			"/usr/bin/buildah push --storage-driver=vfs %s && "+
 			"/usr/bin/buildah push --storage-driver=vfs %s && "+
 			"/usr/bin/buildah push --storage-driver=vfs %s",
-			containerImageRegistry, secretArgsString, context, dockerFileName,
-			imageTag, clusterTypeImageTag, clusterNameImageTag,
-			context,
-			imageTag, clusterTypeImageTag, clusterNameImageTag),
+			cmd, imageTag, clusterTypeImageTag, clusterNameImageTag)
 	}
+
+	return []string{"/bin/bash", "-c", cmd}
 }
 
 func isUsingBuildKit(pipelineInfo *model.PipelineInfo) bool {