implemented ginkgo e2e test for cluster migration

.secrets.baseline

@@ -259,7 +259,7 @@
         "filename": "e2e/e2e_test.go",
         "hashed_secret": "7f38822bc2b03e97325ff310099f457f6f788daf",
         "is_verified": false,
-        "line_number": 300
+        "line_number": 294
       }
     ],
     "internal/dinosaur/pkg/api/public/api/openapi.yaml": [
@@ -312,7 +312,7 @@
         "filename": "pkg/client/fleetmanager/mocks/client_moq.go",
         "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c",
         "is_verified": false,
-        "line_number": 584
+        "line_number": 640
       }
     ],
     "pkg/client/redhatsso/api/api/openapi.yaml": [
@@ -416,5 +416,5 @@
       }
     ]
   },
-  "generated_at": "2024-11-26T16:50:48Z"
+  "generated_at": "2024-12-16T09:41:30Z"
 }

e2e/dns/record_cleanup.go

@@ -0,0 +1,35 @@
+package dns
+
+import (
+	"github.com/aws/aws-sdk-go/service/route53"
+	. "github.com/onsi/gomega"
+	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/api/public"
+	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/services"
+)
+
+// CleanupCentralRequestRecords deletes all route53 resoruces associated with the centralRequest
+func CleanupCentralRequestRecords(route53Client *route53.Route53, centralRequest public.CentralRequest) {
+	dnsLoader := NewRecordsLoader(route53Client, centralRequest)
+	recordSets := dnsLoader.LoadDNSRecords()
+
+	action := string(services.CentralRoutesActionDelete)
+	changes := []*route53.Change{}
+	for _, rs := range recordSets {
+		c := &route53.Change{
+			Action:            &action,
+			ResourceRecordSet: rs,
+		}
+		changes = append(changes, c)
+	}
+
+	if len(changes) == 0 {
+		return
+	}
+
+	_, err := route53Client.ChangeResourceRecordSets(&route53.ChangeResourceRecordSetsInput{
+		HostedZoneId: dnsLoader.rhacsZone.Name,
+		ChangeBatch:  &route53.ChangeBatch{Changes: changes},
+	})
+
+	Expect(err).ToNot(HaveOccurred())
+}

e2e/e2e_canary_upgrade_test.go

@@ -8,6 +8,7 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"github.com/stackrox/acs-fleet-manager/e2e/testutil"
 	"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/central/operator"
 	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/constants"
 	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/api/public"
@@ -61,7 +62,7 @@ var _ = Describe("Fleetshard-sync Targeted Upgrade", Ordered, func() {
 	})
 
 	BeforeEach(func() {
-		SkipIf(!features.TargetedOperatorUpgrades.Enabled() || !runCanaryUpgradeTests, "Skipping canary upgrade test")
+		testutil.SkipIf(!features.TargetedOperatorUpgrades.Enabled() || !runCanaryUpgradeTests, "Skipping canary upgrade test")
 		option := fmImpl.OptionFromEnv()
 		auth, err := fmImpl.NewStaticAuth(ctx, fmImpl.StaticOption{StaticToken: option.Static.StaticToken})
 		Expect(err).ToNot(HaveOccurred())
@@ -249,7 +250,7 @@ var _ = Describe("Fleetshard-sync Targeted Upgrade", Ordered, func() {
 		It("delete central", func() {
 			Expect(deleteCentralByID(ctx, client, createdCentral.Id)).
 				To(Succeed())
-			Eventually(assertCentralRequestDeprovisioning(ctx, client, createdCentral.Id)).
+			Eventually(testutil.AssertCentralRequestDeprovisioning(ctx, client, createdCentral.Id)).
 				WithTimeout(waitTimeout).
 				WithPolling(defaultPolling).
 				Should(Succeed())

e2e/e2e_suite_test.go

@@ -14,9 +14,9 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	openshiftRouteV1 "github.com/openshift/api/route/v1"
+	"github.com/stackrox/acs-fleet-manager/e2e/testutil"
 	"github.com/stackrox/acs-fleet-manager/fleetshard/config"
 	"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/k8s"
-	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/constants"
 	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/api/public"
 	"github.com/stackrox/acs-fleet-manager/pkg/client/fleetmanager"
 	"github.com/stackrox/rox/operator/apis/platform/v1alpha1"
@@ -32,8 +32,8 @@ var (
 	dnsEnabled            bool
 	routesEnabled         bool
 	route53Client         *route53.Route53
-	waitTimeout           = getWaitTimeout()
-	extendedWaitTimeout   = getWaitTimeout() * 3
+	waitTimeout           = testutil.GetWaitTimeout()
+	extendedWaitTimeout   = testutil.GetWaitTimeout() * 3
 	dpCloudProvider       = getEnvDefault("DP_CLOUD_PROVIDER", "standalone")
 	dpRegion              = getEnvDefault("DP_REGION", "standalone")
 	fleetManagerEndpoint  = "http://localhost:8000"
@@ -52,18 +52,6 @@ var (
 	}
 )
 
-func getWaitTimeout() time.Duration {
-	timeoutStr, ok := os.LookupEnv("WAIT_TIMEOUT")
-	if ok {
-		timeout, err := time.ParseDuration(timeoutStr)
-		if err == nil {
-			return timeout
-		}
-		fmt.Printf("Error parsing timeout, using default timeout %v: %s\n", defaultTimeout, err)
-	}
-	return defaultTimeout
-}
-
 func getEnvDefault(key, defaultValue string) string {
 	value, ok := os.LookupEnv(key)
 	if !ok {
@@ -89,7 +77,7 @@ var _ = BeforeSuite(func() {
 	Expect(err).ToNot(HaveOccurred())
 
 	var accessKey, secretKey string
-	dnsEnabled, accessKey, secretKey = isDNSEnabled(routesEnabled)
+	dnsEnabled, accessKey, secretKey = testutil.DNSConfiguration(routesEnabled)
 
 	if dnsEnabled {
 		creds := credentials.NewStaticCredentials(
@@ -125,49 +113,6 @@ func enableTestsGroup(testName string, envName string, defaultValue string) bool
 	return false
 }
 
-func isDNSEnabled(routesEnabled bool) (bool, string, string) {
-	accessKey := os.Getenv("ROUTE53_ACCESS_KEY")
-	secretKey := os.Getenv("ROUTE53_SECRET_ACCESS_KEY")
-	enableExternal := os.Getenv("ENABLE_CENTRAL_EXTERNAL_CERTIFICATE")
-	dnsEnabled := accessKey != "" &&
-		secretKey != "" &&
-		enableExternal != "" && routesEnabled
-	return dnsEnabled, accessKey, secretKey
-}
-
-func assertCentralRequestStatus(ctx context.Context, client *fleetmanager.Client, id string, status string) func() error {
-	return func() error {
-		centralRequest, _, err := client.PublicAPI().GetCentralById(ctx, id)
-		if err != nil {
-			return err
-		}
-		if centralRequest.Status != status {
-			return fmt.Errorf("expected centralRequest status %s, got %s", status, centralRequest.Status)
-		}
-		return nil
-	}
-}
-
-func assertCentralRequestReady(ctx context.Context, client *fleetmanager.Client, id string) func() error {
-	return assertCentralRequestStatus(ctx, client, id, constants.CentralRequestStatusReady.String())
-}
-
-func assertCentralRequestProvisioning(ctx context.Context, client *fleetmanager.Client, id string) func() error {
-	return assertCentralRequestStatus(ctx, client, id, constants.CentralRequestStatusProvisioning.String())
-}
-
-func assertCentralRequestDeprovisioning(ctx context.Context, client *fleetmanager.Client, id string) func() error {
-	return assertCentralRequestStatus(ctx, client, id, constants.CentralRequestStatusDeprovision.String())
-}
-
-func assertCentralRequestDeleting(ctx context.Context, client *fleetmanager.Client, id string) func() error {
-	return assertCentralRequestStatus(ctx, client, id, constants.CentralRequestStatusDeleting.String())
-}
-
-func assertCentralRequestAccepted(ctx context.Context, client *fleetmanager.Client, id string) func() error {
-	return assertCentralRequestStatus(ctx, client, id, constants.CentralRequestStatusAccepted.String())
-}
-
 func obtainCentralRequest(ctx context.Context, client *fleetmanager.Client, id string, request *public.CentralRequest) error {
 	centralRequest, _, err := client.PublicAPI().GetCentralById(ctx, id)
 	if err != nil {
@@ -266,20 +211,6 @@ func assertDeploymentHealthyReplicas(ctx context.Context, namespace, name string
 	}
 }
 
-func assertReencryptIngressRouteExist(ctx context.Context, namespace string, route *openshiftRouteV1.RouteIngress) func() error {
-	return func() error {
-		reencryptIngress, err := routeService.FindReencryptIngress(ctx, namespace)
-		if err != nil {
-			return fmt.Errorf("failed finding reencrypt ingress in namespace %s: %v", namespace, err)
-		}
-		if reencryptIngress == nil {
-			return fmt.Errorf("reencrypt ingress in namespace %s not found", namespace)
-		}
-		*route = *reencryptIngress
-		return nil
-	}
-}
-
 func assertReencryptRouteExist(ctx context.Context, namespace string, route *openshiftRouteV1.Route) func() error {
 	return func() error {
 		reencryptRoute, err := routeService.FindReencryptRoute(ctx, namespace)
@@ -307,9 +238,3 @@ func assertPassthroughRouteExist(ctx context.Context, namespace string, route *o
 		return nil
 	}
 }
-
-func SkipIf(condition bool, message string) {
-	if condition {
-		Skip(message, 1)
-	}
-}

e2e/e2e_test.go

@@ -15,6 +15,7 @@ import (
 	. "github.com/onsi/gomega"
 	openshiftRouteV1 "github.com/openshift/api/route/v1"
 	"github.com/stackrox/acs-fleet-manager/e2e/dns"
+	"github.com/stackrox/acs-fleet-manager/e2e/testutil"
 	"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/k8s"
 	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/constants"
 	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/api/admin/private"
@@ -46,7 +47,6 @@ func newCentralName() string {
 const (
 	defaultPolling = 1 * time.Second
 	skipRouteMsg   = "route resource is not known to test cluster"
-	skipDNSMsg     = "external DNS is not enabled for this test run"
 )
 
 var (
@@ -66,7 +66,7 @@ var _ = Describe("Central", Ordered, func() {
 	})
 
 	BeforeEach(func() {
-		SkipIf(!runCentralTests, "Skipping Central tests")
+		testutil.SkipIf(!runCentralTests, "Skipping Central tests")
 
 		option := fmImpl.OptionFromEnv()
 		auth, err := fmImpl.NewStaticAuth(context.Background(), fmImpl.StaticOption{StaticToken: option.Static.StaticToken})
@@ -124,7 +124,7 @@ var _ = Describe("Central", Ordered, func() {
 		})
 
 		It("should transition central request state to provisioning", func() {
-			Eventually(assertCentralRequestProvisioning(ctx, client, centralRequestID)).
+			Eventually(testutil.AssertCentralRequestProvisioning(ctx, client, centralRequestID)).
 				WithTimeout(waitTimeout).
 				WithPolling(defaultPolling).
 				Should(Succeed())
@@ -156,7 +156,7 @@ var _ = Describe("Central", Ordered, func() {
 
 		// TODO: possible flake. Maybe this test will be executed after the routes are created
 		It("should not expose URLs until the routes are created", func() {
-			SkipIf(!routesEnabled, skipRouteMsg)
+			testutil.SkipIf(!routesEnabled, skipRouteMsg)
 			var centralRequest public.CentralRequest
 			Expect(obtainCentralRequest(ctx, client, centralRequestID, &centralRequest)).
 				To(Succeed())
@@ -165,14 +165,14 @@ var _ = Describe("Central", Ordered, func() {
 		})
 
 		It("should transition central request state to ready", func() {
-			Eventually(assertCentralRequestReady(ctx, client, centralRequestID)).
+			Eventually(testutil.AssertCentralRequestReady(ctx, client, centralRequestID)).
 				WithTimeout(extendedWaitTimeout).
 				WithPolling(defaultPolling).
 				Should(Succeed())
 		})
 
 		It("should have created central routes", func() {
-			SkipIf(!routesEnabled, skipRouteMsg)
+			testutil.SkipIf(!routesEnabled, skipRouteMsg)
 
 			var centralRequest public.CentralRequest
 			Expect(obtainCentralRequest(ctx, client, centralRequestID, &centralRequest)).
@@ -204,14 +204,14 @@ var _ = Describe("Central", Ordered, func() {
 		})
 
 		It("should have created AWS Route53 records", func() {
-			SkipIf(!dnsEnabled, skipDNSMsg)
+			testutil.SkipIf(!dnsEnabled, testutil.SkipDNSMsg)
 
 			var centralRequest public.CentralRequest
 			Expect(obtainCentralRequest(ctx, client, centralRequestID, &centralRequest)).
 				To(Succeed())
 
 			var reencryptIngress openshiftRouteV1.RouteIngress
-			Eventually(assertReencryptIngressRouteExist(ctx, namespaceName, &reencryptIngress)).
+			Eventually(testutil.AssertReencryptIngressRouteExist(context.Background(), routeService, namespaceName, &reencryptIngress)).
 				WithTimeout(waitTimeout).
 				WithPolling(defaultPolling).
 				Should(Succeed())
@@ -224,13 +224,7 @@ var _ = Describe("Central", Ordered, func() {
 				Should(HaveLen(len(dnsRecordsLoader.CentralDomainNames)), "Started at %s", time.Now())
 
 			recordSets := dnsRecordsLoader.LastResult
-			for idx, domain := range dnsRecordsLoader.CentralDomainNames {
-				recordSet := recordSets[idx]
-				Expect(len(recordSet.ResourceRecords)).To(Equal(1))
-				record := recordSet.ResourceRecords[0]
-				Expect(*recordSet.Name).To(Equal(domain))
-				Expect(*record.Value).To(Equal(reencryptIngress.RouterCanonicalHostname)) // TODO use route specific ingress instead of comparing with reencryptIngress for all cases
-			}
+			testutil.AssertDNSMatchesRouter(dnsRecordsLoader.CentralDomainNames, recordSets, &reencryptIngress)
 		})
 
 		It("should backup important secrets in FM database", func() {
@@ -338,7 +332,7 @@ var _ = Describe("Central", Ordered, func() {
 		It("should transition central to deprovisioning state", func() {
 			Expect(deleteCentralByID(ctx, client, centralRequestID)).
 				To(Succeed())
-			Eventually(assertCentralRequestDeprovisioning(ctx, client, centralRequestID)).
+			Eventually(testutil.AssertCentralRequestDeprovisioning(ctx, client, centralRequestID)).
 				WithTimeout(waitTimeout).
 				WithPolling(defaultPolling).
 				Should(Succeed())
@@ -359,7 +353,7 @@ var _ = Describe("Central", Ordered, func() {
 		})
 
 		It("should delete external DNS entries", func() {
-			SkipIf(!dnsEnabled, skipDNSMsg)
+			testutil.SkipIf(!dnsEnabled, testutil.SkipDNSMsg)
 			var centralRequest public.CentralRequest
 			Expect(obtainCentralRequest(ctx, client, centralRequestID, &centralRequest)).
 				To(Succeed())
@@ -409,7 +403,7 @@ var _ = Describe("Central", Ordered, func() {
 		})
 
 		It("should transition central's state to ready", func() {
-			Eventually(assertCentralRequestReady(ctx, client, centralRequestID)).
+			Eventually(testutil.AssertCentralRequestReady(ctx, client, centralRequestID)).
 				WithTimeout(waitTimeout).
 				WithPolling(defaultPolling).
 				Should(Succeed())
@@ -418,7 +412,7 @@ var _ = Describe("Central", Ordered, func() {
 		It("should transition central to deprovisioning state when deleting", func() {
 			Expect(deleteCentralByID(ctx, client, centralRequestID)).
 				To(Succeed())
-			Eventually(assertCentralRequestDeprovisioning(ctx, client, centralRequestID)).
+			Eventually(testutil.AssertCentralRequestDeprovisioning(ctx, client, centralRequestID)).
 				WithTimeout(waitTimeout).
 				WithPolling(defaultPolling).
 				Should(Succeed())
@@ -439,7 +433,7 @@ var _ = Describe("Central", Ordered, func() {
 		})
 
 		It("should delete external DNS entries", func() {
-			SkipIf(!dnsEnabled, skipDNSMsg)
+			testutil.SkipIf(!dnsEnabled, testutil.SkipDNSMsg)
 			var centralRequest public.CentralRequest
 			Expect(obtainCentralRequest(ctx, client, centralRequestID, &centralRequest)).
 				To(Succeed())
@@ -481,7 +475,7 @@ var _ = Describe("Central", Ordered, func() {
 		var readyCentralRequest public.CentralRequest
 
 		It("should transition central's state to ready", func() {
-			Eventually(assertCentralRequestReady(ctx, client, centralRequestID)).
+			Eventually(testutil.AssertCentralRequestReady(ctx, client, centralRequestID)).
 				WithTimeout(extendedWaitTimeout).
 				WithPolling(defaultPolling).
 				Should(Succeed())
@@ -520,7 +514,7 @@ var _ = Describe("Central", Ordered, func() {
 		})
 
 		It("should delete external DNS entries", func() {
-			SkipIf(!dnsEnabled, skipDNSMsg)
+			testutil.SkipIf(!dnsEnabled, testutil.SkipDNSMsg)
 			dnsRecordsLoader := dns.NewRecordsLoader(route53Client, readyCentralRequest)
 			Eventually(dnsRecordsLoader.LoadDNSRecords).
 				WithTimeout(waitTimeout).