Use E2E config instead of ginkgo flags (#657)

* Use E2E config instead of ginkgo flags

Signed-off-by: Alexandr Demicev <[email protected]>

* Add documentation for public E2E structures and functions

Signed-off-by: Alexandr Demicev <[email protected]>

* Use rancher turtles version from the config file

Signed-off-by: Alexandr Demicev <[email protected]>

* Update README with information about new config options

Signed-off-by: Alexandr Demicev <[email protected]>

---------

Signed-off-by: Alexandr Demicev <[email protected]>

.github/workflows/e2e-long.yaml

@@ -15,7 +15,7 @@ jobs:
       test_name: Import via GitOps
       run_azure_janitor: false
       artifact_name: artifacts_import_gitops
-      management_cluster_infrastructure: eks
+      MANAGEMENT_CLUSTER_ENVIRONMENT: eks
     secrets: inherit
   e2e_import_gitops_v3:
     uses: ./.github/workflows/run-e2e-suite.yaml
@@ -24,7 +24,7 @@ jobs:
       test_name: Import via GitOps [v3]
       run_azure_janitor: false
       artifact_name: artifacts_import_gitops_v3
-      management_cluster_infrastructure: eks
+      MANAGEMENT_CLUSTER_ENVIRONMENT: eks
     secrets: inherit
   e2e_v2prov:
     uses: ./.github/workflows/run-e2e-suite.yaml
@@ -33,7 +33,7 @@ jobs:
       test_name: v2 provisioning
       run_azure_janitor: true
       artifact_name: artifacts_v2prov
-      management_cluster_infrastructure: eks
+      MANAGEMENT_CLUSTER_ENVIRONMENT: eks
     secrets: inherit
   e2e_update_labels:
     uses: ./.github/workflows/run-e2e-suite.yaml
@@ -42,7 +42,7 @@ jobs:
       test_name: Update labels
       run_azure_janitor: true
       artifact_name: artifacts_update_labels
-      management_cluster_infrastructure: eks
+      MANAGEMENT_CLUSTER_ENVIRONMENT: eks
     secrets: inherit
   e2e_embedded_capi_disabled:
     uses: ./.github/workflows/run-e2e-suite.yaml
@@ -51,7 +51,7 @@ jobs:
       test_name: Embedded CAPI disabled
       run_azure_janitor: false
       artifact_name: artifacts_embedded_capi
-      management_cluster_infrastructure: eks
+      MANAGEMENT_CLUSTER_ENVIRONMENT: eks
     secrets: inherit
   e2e_embedded_capi_disabled_v3:
     uses: ./.github/workflows/run-e2e-suite.yaml
@@ -60,5 +60,5 @@ jobs:
       test_name: Embedded CAPI disabled [v3]
       run_azure_janitor: false
       artifact_name: artifacts_embedded_capi_v3
-      management_cluster_infrastructure: eks
+      MANAGEMENT_CLUSTER_ENVIRONMENT: eks
     secrets: inherit

.github/workflows/e2e-short-test.yaml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
 
 env:
-  MANAGEMENT_CLUSTER_INFRASTRUCTURE: "isolated-kind"
+  MANAGEMENT_CLUSTER_ENVIRONMENT: "isolated-kind"
   GINKGO_LABEL_FILTER: "short"
 
 jobs:

.github/workflows/e2e-short.yaml

@@ -5,7 +5,7 @@ on:
     types: [opened, edited, synchronize, reopened, labeled, unlabeled]
 
 env:
-  MANAGEMENT_CLUSTER_INFRASTRUCTURE: "isolated-kind"
+  MANAGEMENT_CLUSTER_ENVIRONMENT: "isolated-kind"
   GINKGO_LABEL_FILTER: "short"
 
 jobs:

.github/workflows/run-e2e-suite.yaml

@@ -1,7 +1,7 @@
 on:
   workflow_call:
     inputs:
-      management_cluster_infrastructure:
+      MANAGEMENT_CLUSTER_ENVIRONMENT:
         description: "The infrastructure to use for the management cluster: eks, kind or isolated-kind"
         type: string
         required: true
@@ -43,7 +43,7 @@ env:
   AWS_REGION: eu-west-2
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-  MANAGEMENT_CLUSTER_INFRASTRUCTURE: ${{ inputs.management_cluster_infrastructure }}
+  MANAGEMENT_CLUSTER_ENVIRONMENT: ${{ inputs.MANAGEMENT_CLUSTER_ENVIRONMENT }}
   GINKGO_LABEL_FILTER: full 
   GINKGO_TESTS: ${{ github.workspace }}/${{ inputs.test_suite }}
 
@@ -84,7 +84,7 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: build and push e2e image
-        if: ${{ inputs.management_cluster_infrastructure == 'eks' }}
+        if: ${{ inputs.MANAGEMENT_CLUSTER_ENVIRONMENT == 'eks' }}
         run: make e2e-image-push
       - name: Run e2e tests
         run: make test-e2e

Makefile

@@ -80,8 +80,15 @@ GINKGO_NOCOLOR ?= false
 GINKGO_LABEL_FILTER ?= short
 GINKGO_TESTS ?= $(ROOT_DIR)/$(TEST_DIR)/e2e/suites/...
 
-MANAGEMENT_CLUSTER_INFRASTRUCTURE ?= eks
-E2ECONFIG_VARS ?= MANAGEMENT_CLUSTER_INFRASTRUCTURE=$(MANAGEMENT_CLUSTER_INFRASTRUCTURE)
+MANAGEMENT_CLUSTER_ENVIRONMENT ?= eks
+
+E2ECONFIG_VARS ?= MANAGEMENT_CLUSTER_ENVIRONMENT=$(MANAGEMENT_CLUSTER_ENVIRONMENT) \
+ARTIFACTS=$(ARTIFACTS) \
+HELM_BINARY_PATH=$(HELM) \
+CLUSTERCTL_BINARY_PATH=$(CLUSTERCTL) \
+SKIP_RESOURCE_CLEANUP=$(SKIP_RESOURCE_CLEANUP) \
+USE_EXISTING_CLUSTER=$(USE_EXISTING_CLUSTER) \
+TURTLES_PATH=$(ROOT_DIR)/$(CHART_RELEASE_DIR)
 
 # to set multiple ginkgo skip flags, if any
 ifneq ($(strip $(GINKGO_SKIP)),)
@@ -535,14 +542,7 @@ test-e2e: $(GINKGO) $(HELM) $(CLUSTERCTL) kubectl e2e-image ## Run the end-to-en
 		-poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" --label-filter="$(GINKGO_LABEL_FILTER)" \
 		$(_SKIP_ARGS) --nodes=$(GINKGO_NODES) --timeout=$(GINKGO_TIMEOUT) --no-color=$(GINKGO_NOCOLOR) \
 		--output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) $(GINKGO_TESTS) -- \
-	    -e2e.artifacts-folder="$(ARTIFACTS)" \
-	    -e2e.config="$(E2E_CONF_FILE)" \
-		-e2e.helm-binary-path=$(HELM) \
-		-e2e.clusterctl-binary-path=$(CLUSTERCTL) \
-		-e2e.chart-path=$(ROOT_DIR)/$(CHART_RELEASE_DIR) \
-	    -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \
-		-e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) \
-		-e2e.gitea-custom-ingress=$(GITEA_CUSTOM_INGRESS)
+	    -e2e.config="$(E2E_CONF_FILE)"
 
 .PHONY: e2e-image
 e2e-image: ## Build the image for e2e tests

test/e2e/README.md

@@ -99,6 +99,7 @@ variables:
   RANCHER_HOSTNAME: "localhost" # Your ngrok domain
   NGROK_API_KEY: "" # Key and token values for establishing ingress
   NGROK_AUTHTOKEN: ""
+  MANAGEMENT_CLUSTER_ENVIRONMENT: "isolated-kind" # Environment to run the tests in: eks, isolated-kind, kind.
 ```
 
 ## Testdata
@@ -113,7 +114,19 @@ While all the tests are based on the combination of [ginkgo](https://github.com/
 
 ## Cluster configuration
 
-[Kind](https://kind.sigs.k8s.io/) is used to set up a cluster for e2e tests. All required components like rancher, rancher-turtles and [cluster-api-operator](https://github.com/kubernetes-sigs/cluster-api-operator) (which provisions cluster-api with required providers) are installed using [helm](https://kind.sigs.k8s.io/) charts.
+### Kind
+
+[Kind](https://kind.sigs.k8s.io/) is used to set up a cluster for e2e tests. All required components like rancher, rancher-turtles and [cluster-api-operator](https://github.com/kubernetes-sigs/cluster-api-operator) (which provisions cluster-api with required providers) are installed using [helm](https://kind.sigs.k8s.io/) charts. This option can be enabled by setting `MANAGEMENT_CLUSTER_ENVIRONMENT` to `kind`. It's also required to set `NGROK_API_KEY`, `NGROK_AUTHTOKEN` and `RANCHER_HOSTNAME` environment variables.
+
+### Isolated Kind
+
+This is similar to Kind but instead of public endpoint for Rancher, it uses the internal IP of CP node. This setup can be used to test providers are running in the same network as Rancher. This option can be enabled by setting `MANAGEMENT_CLUSTER_ENVIRONMENT` to `isolated-kind`.
+
+### EKS
+
+EKS is used to set up a cluster for e2e tests. In this setup nginx ingress will be deployed to provide a public endpoint for Rancher. This option can be enabled by setting `MANAGEMENT_CLUSTER_ENVIRONMENT` to `eks`.
+
+### Customizing the cluster
 
 To configure individual components, a series of `server-side-apply` patches are being issued. All required patch manifests are located under `test/e2e/resources/config`. Under circumstances each manifest could have a limited environment based configuration with `envsubst` (for example: setting `RANCHER_HOSTNAME` value in ingress configuration).
 

test/e2e/config/operator.yaml

@@ -27,22 +27,32 @@ intervals:
   default/wait-turtles-uninstall: ["10m", "30s"]
 
 variables:
-  MANAGEMENT_CLUSTER_INFRASTRUCTURE: "isolated-kind" # supported options are eks, isolated-kind, kind
-  RANCHER_VERSION: "v2.8.1"
+  MANAGEMENT_CLUSTER_ENVIRONMENT: "isolated-kind" # supported options are eks, isolated-kind, kind
   RANCHER_ALPHA_VERSION: "v2.9.1-alpha1"
+  CLUSTERCTL_BINARY_PATH: ""
+  USE_EXISTING_CLUSTER: "false"
+  SKIP_RESOURCE_CLEANUP: "false"
+  ARTIFACTS_FOLDER: "_artifacts"
+  HELM_BINARY_PATH: "helm"
+  HELM_EXTRA_VALUES_FOLDER: "/tmp"
   KUBERNETES_VERSION: "v1.28.6"
   KUBERNETES_MANAGEMENT_VERSION: "v1.27.0"
+  RKE2_VERSION: "v1.28.1+rke2r1"
   KUBERNETES_MANAGEMENT_AWS_REGION: "eu-west-2"
   RANCHER_HOSTNAME: "localhost"
   RANCHER_FEATURES: ""
   RANCHER_PATH: "rancher-latest/rancher"
   RANCHER_ALPHA_PATH: "rancher-alpha/rancher"
+  RANCHER_ALPHA_URL: "https://releases.rancher.com/server-charts/alpha"
+  RANCHER_VERSION: "v2.8.1"
+  TURTLES_VERSION: "v0.0.1"
+  TURTLES_PATH: "turtles/rancher-turtles"
+  TURTLES_REPO_NAME: "turtles"
+  TURTLES_URL: https://rancher.github.io/turtles
   CPI_IMAGE_K8S_VERSION: "v1.27.0"
-  RKE2_VERSION: "v1.28.1+rke2r1"
   RANCHER_REPO_NAME: "rancher-latest"
   RANCHER_ALPHA_REPO_NAME: "rancher-alpha"
   RANCHER_URL: "https://releases.rancher.com/server-charts/latest"
-  RANCHER_ALPHA_URL: "https://releases.rancher.com/server-charts/alpha"
   CERT_MANAGER_URL: "https://charts.jetstack.io"
   CERT_MANAGER_REPO_NAME: "jetstack"
   CERT_MANAGER_PATH: "jetstack/cert-manager"

test/e2e/const.go

@@ -98,16 +98,16 @@ const (
 	NginxIngressDeployment  = "ingress-nginx-controller"
 )
 
-type ManagementClusterInfrastuctureType string
+type ManagementClusterEnvironmentType string
 
 const (
-	ManagementClusterInfrastuctureEKS          ManagementClusterInfrastuctureType = "eks"
-	ManagementClusterInfrastuctureIsolatedKind ManagementClusterInfrastuctureType = "isolated-kind"
-	ManagementClusterInfrastuctureKind         ManagementClusterInfrastuctureType = "kind"
+	ManagementClusterEnvironmentEKS          ManagementClusterEnvironmentType = "eks"
+	ManagementClusterEnvironmentIsolatedKind ManagementClusterEnvironmentType = "isolated-kind"
+	ManagementClusterEnvironmentKind         ManagementClusterEnvironmentType = "kind"
 )
 
 const (
-	ManagementClusterInfrastucture = "MANAGEMENT_CLUSTER_INFRASTRUCTURE"
+	ManagementClusterEnvironmentVar = "MANAGEMENT_CLUSTER_ENVIRONMENT"
 
 	KubernetesManagementVersionVar = "KUBERNETES_MANAGEMENT_VERSION"
 
@@ -141,6 +141,17 @@ const (
 	GiteaUserNameVar     = "GITEA_USER_NAME"
 	GiteaUserPasswordVar = "GITEA_USER_PWD"
 
+	ArtifactsFolderVar       = "ARTIFACTS_FOLDER"
+	UseExistingClusterVar    = "USE_EXISTING_CLUSTER"
+	HelmBinaryPathVar        = "HELM_BINARY_PATH"
+	HelmExtraValuesFolderVar = "HELM_EXTRA_VALUES_FOLDER"
+	TurtlesVersionVar        = "TURTLES_VERSION"
+	TurtlesPathVar           = "TURTLES_PATH"
+	TurtlesUrlVar            = "TURTLES_URL"
+	TurtlesRepoNameVar       = "TURTLES_REPO_NAME"
+	SkipResourceCleanupVar   = "SKIP_RESOURCE_CLEANUP"
+	ClusterctlBinaryPathVar  = "CLUSTERCTL_BINARY_PATH"
+
 	RKE2VersionVar = "RKE2_VERSION"
 
 	CapaEncodedCredentialsVar = "CAPA_ENCODED_CREDS"

test/e2e/flags.go

@@ -26,41 +26,9 @@ import (
 type FlagValues struct {
 	// ConfigPath is the path to the e2e config file.
 	ConfigPath string
-
-	// UseExistingCluster instructs the test to use the current cluster instead of creating a new one (default discovery rules apply).
-	UseExistingCluster bool
-
-	// ArtifactFolder is the folder to store e2e test artifacts.
-	ArtifactFolder string
-
-	// SkipCleanup prevents cleanup of test resources e.g. for debug purposes.
-	SkipCleanup bool
-
-	// HelmBinaryPath is the path to the helm binary.
-	HelmBinaryPath string
-
-	// HelmExtraValuesDir is the location where extra values files will be stored.
-	HelmExtraValuesDir string
-
-	// ChartPath is the path to the operator chart.
-	ChartPath string
-
-	// ClusterctlBinaryPath is the path to the clusterctl binary to use.
-	ClusterctlBinaryPath string
-
-	// GiteaCustomIngress is the flag to enable custom ingress for Gitea.
-	GiteaCustomIngress bool
 }
 
 // InitFlags is used to specify the standard flags for the e2e tests.
 func InitFlags(values *FlagValues) {
 	flag.StringVar(&values.ConfigPath, "e2e.config", "config/operator.yaml", "path to the e2e config file")
-	flag.StringVar(&values.ArtifactFolder, "e2e.artifacts-folder", "_artifacts", "folder where e2e test artifact should be stored")
-	flag.BoolVar(&values.SkipCleanup, "e2e.skip-resource-cleanup", false, "if true, the resource cleanup after tests will be skipped")
-	flag.BoolVar(&values.UseExistingCluster, "e2e.use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)")
-	flag.StringVar(&values.HelmBinaryPath, "e2e.helm-binary-path", "helm", "path to the helm binary")
-	flag.StringVar(&values.HelmExtraValuesDir, "e2e.helm-extra-values-path", "/tmp", "path to the extra values file")
-	flag.StringVar(&values.ClusterctlBinaryPath, "e2e.clusterctl-binary-path", "helm", "path to the clusterctl binary")
-	flag.StringVar(&values.ChartPath, "e2e.chart-path", "", "path to the operator chart")
-	flag.BoolVar(&values.GiteaCustomIngress, "e2e.gitea-custom-ingress", false, "if true, the test will use a custom ingress for Gitea")
 }

test/e2e/helpers.go

@@ -25,6 +25,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strconv"
 
 	. "github.com/onsi/gomega"
 
@@ -115,3 +116,15 @@ func CreateClusterctlLocalRepository(ctx context.Context, config *clusterctl.E2E
 	Expect(clusterctlConfig).To(BeAnExistingFile(), "The clusterctl config file does not exists in the local repository %s", repositoryFolder)
 	return clusterctlConfig
 }
+
+func ValidateE2EConfig(config *clusterctl.E2EConfig) {
+	Expect(os.MkdirAll(config.GetVariable(ArtifactsFolderVar), 0o755)).To(Succeed(), "Invalid test suite argument. Can't create artifacts folder %q", config.GetVariable(ArtifactsFolderVar))
+	Expect(config.GetVariable(HelmBinaryPathVar)).To(BeAnExistingFile(), "Invalid test suite argument. HELM_BINARY_PATH should be an existing file.")
+	Expect(config.GetVariable(TurtlesPathVar)).To(BeAnExistingFile(), "Invalid test suite argument. TURTLES_PATH should be an existing file.")
+
+	_, err := strconv.ParseBool(config.GetVariable(UseExistingClusterVar))
+	Expect(err).ToNot(HaveOccurred(), "Invalid test suite argument. Can't parse USE_EXISTING_CLUSTER %q", config.GetVariable(UseExistingClusterVar))
+
+	_, err = strconv.ParseBool(config.GetVariable(SkipResourceCleanupVar))
+	Expect(err).ToNot(HaveOccurred(), "Invalid test suite argument. Can't parse SKIP_RESOURCE_CLEANUP %q", config.GetVariable(SkipResourceCleanupVar))
+}

test/e2e/specs/migrate_gitops_provv1_mgmtv3.go

@@ -353,6 +353,7 @@ func MigrateToV3UsingGitOpsSpec(ctx context.Context, inputGetter func() MigrateT
 		testenv.DeployChartMuseum(ctx, testenv.DeployChartMuseumInput{
 			HelmBinaryPath:        input.HelmBinaryPath,
 			ChartsPath:            input.ChartPath,
+			ChartVersion:          input.E2EConfig.GetVariable(e2e.TurtlesVersionVar),
 			BootstrapClusterProxy: input.BootstrapClusterProxy,
 			WaitInterval:          input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-controllers"),
 		})
@@ -362,7 +363,7 @@ func MigrateToV3UsingGitOpsSpec(ctx context.Context, inputGetter func() MigrateT
 			HelmBinaryPath:               input.HelmBinaryPath,
 			Namespace:                    turtlesframework.DefaultRancherTurtlesNamespace,
 			Image:                        fmt.Sprintf("ghcr.io/rancher/turtles-e2e-%s", runtime.GOARCH),
-			Tag:                          "v0.0.1",
+			Tag:                          input.E2EConfig.GetVariable(e2e.TurtlesVersionVar),
 			WaitDeploymentsReadyInterval: input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-controllers"),
 			SkipCleanup:                  true,
 			AdditionalValues:             map[string]string{},

test/e2e/suites/embedded-capi-disabled-v3/embedded_capi_disabled_v3_test.go

@@ -40,8 +40,8 @@ var _ = Describe("[AWS] [EKS] [management.cattle.io/v3] Create and delete CAPI c
 			E2EConfig:                      e2eConfig,
 			BootstrapClusterProxy:          setupClusterResult.BootstrapClusterProxy,
 			ClusterctlConfigPath:           flagVals.ConfigPath,
-			ClusterctlBinaryPath:           flagVals.ClusterctlBinaryPath,
-			ArtifactFolder:                 flagVals.ArtifactFolder,
+			ClusterctlBinaryPath:           e2eConfig.GetVariable(e2e.ClusterctlBinaryPathVar),
+			ArtifactFolder:                 artifactsFolder,
 			ClusterTemplate:                e2e.CAPIAwsEKSMMP,
 			ClusterName:                    "highlander-e2e-cluster1",
 			ControlPlaneMachineCount:       ptr.To[int](1),