diff --git a/deploy/host-operator/e2e-tests/toolchainconfig.yaml b/deploy/host-operator/e2e-tests/toolchainconfig.yaml index 9161aeba2..279eb27da 100644 --- a/deploy/host-operator/e2e-tests/toolchainconfig.yaml +++ b/deploy/host-operator/e2e-tests/toolchainconfig.yaml @@ -29,6 +29,9 @@ spec: spaceBindingRequestEnabled: true tiers: durationBeforeChangeTierRequestDeletion: '5s' + featureToggles: + - name: "test-feature" + weight: 100 toolchainStatus: toolchainStatusRefreshTime: '1s' members: diff --git a/go.mod b/go.mod index e14ef938c..fcce71265 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ module github.com/codeready-toolchain/toolchain-e2e require ( - github.com/codeready-toolchain/api v0.0.0-20240708122235-0af5a9a178bb + github.com/codeready-toolchain/api v0.0.0-20240717145630-bb67a632867a github.com/codeready-toolchain/toolchain-common v0.0.0-20240716065433-8604fe46b96a github.com/davecgh/go-spew v1.1.1 github.com/fatih/color v1.12.0 diff --git a/go.sum b/go.sum index 309b78791..8ffa62b72 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/codeready-toolchain/api v0.0.0-20240708122235-0af5a9a178bb h1:Wc9CMsv0ODZv9dM5qF3OI0mFDO95YNIXV/8oRvoz8aE= -github.com/codeready-toolchain/api v0.0.0-20240708122235-0af5a9a178bb/go.mod h1:ie9p4LenCCS0LsnbWp6/xwpFDdCWYE0KWzUO6Sk1g0E= +github.com/codeready-toolchain/api v0.0.0-20240717145630-bb67a632867a h1:La7GOCysmkU+4vnN8lDzXFJwJiA1LWZ9YkX/yQXYnpw= +github.com/codeready-toolchain/api v0.0.0-20240717145630-bb67a632867a/go.mod h1:ie9p4LenCCS0LsnbWp6/xwpFDdCWYE0KWzUO6Sk1g0E= github.com/codeready-toolchain/toolchain-common v0.0.0-20240716065433-8604fe46b96a h1:HcaJtZCLfYkWZCxIa3iTvq3zgn711JGqPLkunBTfGSc= github.com/codeready-toolchain/toolchain-common v0.0.0-20240716065433-8604fe46b96a/go.mod h1:8M9k7w2VSyRKSK6P08Jo2ddW3uyGgxCcSitnYa3HK9o= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= diff --git a/test/e2e/parallel/nstemplatetier_test.go b/test/e2e/parallel/nstemplatetier_test.go index f21ca22ac..1c5b4c888 100644 --- a/test/e2e/parallel/nstemplatetier_test.go +++ b/test/e2e/parallel/nstemplatetier_test.go @@ -1,8 +1,12 @@ package parallel import ( + "bytes" "context" "fmt" + "html/template" + v1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/runtime" "testing" "time" @@ -303,3 +307,125 @@ func TestKsctlGeneratedTiers(t *testing.T) { }) } } + +func TestFeatureToggles(t *testing.T) { + t.Parallel() + awaitilities := WaitForDeployments(t) + hostAwait := awaitilities.Host() + memberAwait := awaitilities.Member1() + + base1nsTier, err := hostAwait.WaitForNSTemplateTier(t, "base1ns") + require.NoError(t, err) + + t.Run("provision space with enabled feature", func(t *testing.T) { + // given + + // Create a new tier which is a copy of base1ns but with an additional ClusterRoleBinding object with "test-feature" annotation. + // "feature-test" feature is defined in the ToolchainConfig and has 100 weight + tier := tiers.CreateCustomNSTemplateTier(t, hostAwait, "ftier", base1nsTier, + withClusterRoleBinding(t, base1nsTier, "test-feature"), + tiers.WithNamespaceResources(t, base1nsTier), + tiers.WithSpaceRoles(t, base1nsTier)) + _, err := hostAwait.WaitForNSTemplateTier(t, tier.Name) + require.NoError(t, err) + + // when + + // Now let's create a Space + user := NewSignupRequest(awaitilities). + Username("featured-user"). + Email("featured@domain.com"). + ManuallyApprove(). + EnsureMUR(). + SpaceTier("base1ns"). + TargetCluster(memberAwait). + RequireConditions(wait.ConditionSet(wait.Default(), wait.ApprovedByAdmin())...). + Execute(t) + // and promote that space to the ftier tier + tiers.MoveSpaceToTier(t, hostAwait, "featured-user", tier.Name) + VerifyResourcesProvisionedForSpaceWithCustomTier(t, hostAwait, memberAwait, "featured-user", tier) + + // then + + // Verify that the space has the feature annotation - the weight is set to 100, so it should be added to all Spaces in all tiers + space, err := hostAwait.WaitForSpace(t, user.Space.Name) + require.NoError(t, err) + require.NotEmpty(t, space.Annotations) + assert.Equal(t, "test-feature", space.Annotations[toolchainv1alpha1.FeatureToggleNameAnnotationKey]) + // and CRB for the that feature has been created + crbName := fmt.Sprintf("%s-%s", user.Space.Name, "test-feature") + _, err = wait.For(t, memberAwait.Awaitility, &v1.ClusterRoleBinding{}).WithNameThat(crbName) + require.NoError(t, err) + + t.Run("disable feature", func(t *testing.T) { + // when + + // Now let's disable the feature for the Space by removing the feature annotation + _, err := hostAwait.UpdateSpace(t, user.Space.Name, func(s *toolchainv1alpha1.Space) { + delete(s.Annotations, toolchainv1alpha1.FeatureToggleNameAnnotationKey) + }) + require.NoError(t, err) + + // then + err = wait.For(t, memberAwait.Awaitility, &v1.ClusterRoleBinding{}).WithNameDeleted(crbName) + require.NoError(t, err) + + t.Run("re-enable feature", func(t *testing.T) { + // when + + // Now let's re-enable the feature for the Space by restoring the feature annotation + _, err := hostAwait.UpdateSpace(t, user.Space.Name, func(s *toolchainv1alpha1.Space) { + if s.Annotations == nil { + s.Annotations = make(map[string]string) + } + s.Annotations[toolchainv1alpha1.FeatureToggleNameAnnotationKey] = "test-feature" + }) + require.NoError(t, err) + + // then + // Verify that the CRB is back + _, err = wait.For(t, memberAwait.Awaitility, &v1.ClusterRoleBinding{}).WithNameThat(crbName) + require.NoError(t, err) + }) + }) + }) +} + +func withClusterRoleBinding(t *testing.T, otherTier *toolchainv1alpha1.NSTemplateTier, feature string) tiers.CustomNSTemplateTierModifier { + var tpl bytes.Buffer + err := template.Must(template.New("crb").Parse(viewCRB)).Execute(&tpl, map[string]interface{}{ + "featureName": feature, + }) + require.NoError(t, err) + + return tiers.WithClusterResources(t, otherTier, func(template *toolchainv1alpha1.TierTemplate) error { + clusterRB := runtime.RawExtension{ + Raw: tpl.Bytes(), + } + template.Spec.Template.Objects = append(template.Spec.Template.Objects, clusterRB) + return nil + }) +} + +var viewCRB = `{ + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "ClusterRoleBinding", + "metadata": { + "name": "${SPACE_NAME}-{{ .featureName }}", + "annotations": { + "toolchain.dev.openshift.com/feature": "{{ .featureName }}" + } + }, + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "ClusterRole", + "name": "view" + }, + "subjects": [ + { + "kind": "User", + "name": "${USERNAME}" + } + ] +} +` diff --git a/testsupport/tiers/tier_setup.go b/testsupport/tiers/tier_setup.go index 447416985..c95eba799 100644 --- a/testsupport/tiers/tier_setup.go +++ b/testsupport/tiers/tier_setup.go @@ -29,11 +29,13 @@ type CustomNSTemplateTier struct { type CustomNSTemplateTierModifier func(*HostAwaitility, *CustomNSTemplateTier) error -func WithClusterResources(t *testing.T, otherTier *toolchainv1alpha1.NSTemplateTier) CustomNSTemplateTierModifier { +type TierTemplateModifier func(*toolchainv1alpha1.TierTemplate) error + +func WithClusterResources(t *testing.T, otherTier *toolchainv1alpha1.NSTemplateTier, modifiers ...TierTemplateModifier) CustomNSTemplateTierModifier { return func(hostAwait *HostAwaitility, tier *CustomNSTemplateTier) error { tier.ClusterResourcesTier = otherTier // configure the "wrapped" NSTemplateTier - tmplRef, err := duplicateTierTemplate(t, hostAwait, otherTier.Namespace, tier.Name, otherTier.Spec.ClusterResources.TemplateRef) + tmplRef, err := duplicateTierTemplate(t, hostAwait, otherTier.Namespace, tier.Name, otherTier.Spec.ClusterResources.TemplateRef, modifiers...) if err != nil { return err } @@ -78,6 +80,9 @@ func WithSpaceRoles(t *testing.T, otherTier *toolchainv1alpha1.NSTemplateTier) C } } +// CreateCustomNSTemplateTier creates a custom tier. +// If no modifiers provided then the new tier will use copies of the baseTier cluster, namespace and space roles templates +// without any modifications. func CreateCustomNSTemplateTier(t *testing.T, hostAwait *HostAwaitility, name string, baseTier *toolchainv1alpha1.NSTemplateTier, modifiers ...CustomNSTemplateTierModifier) *CustomNSTemplateTier { tier := &CustomNSTemplateTier{ NSTemplateTier: &toolchainv1alpha1.NSTemplateTier{ @@ -91,12 +96,14 @@ func CreateCustomNSTemplateTier(t *testing.T, hostAwait *HostAwaitility, name st }, }, } - // add default values before custom values... - modifiers = append([]CustomNSTemplateTierModifier{ - WithClusterResources(t, baseTier), - WithNamespaceResources(t, baseTier), - WithSpaceRoles(t, baseTier), - }, modifiers...) + if len(modifiers) == 0 { + // If no modifiers provided then use default modifiers which would use resources from the base tier. + modifiers = []CustomNSTemplateTierModifier{ + WithClusterResources(t, baseTier), + WithNamespaceResources(t, baseTier), + WithSpaceRoles(t, baseTier), + } + } // ... and apply for _, modify := range modifiers { @@ -108,7 +115,7 @@ func CreateCustomNSTemplateTier(t *testing.T, hostAwait *HostAwaitility, name st return tier } -// createCustomNSTemplateTier updates the given "tier" using the modifiers +// UpdateCustomNSTemplateTier updates the given "tier" using the modifiers // returns the latest version of the NSTemplateTier func UpdateCustomNSTemplateTier(t *testing.T, hostAwait *HostAwaitility, tier *CustomNSTemplateTier, modifiers ...CustomNSTemplateTierModifier) *CustomNSTemplateTier { // reload the underlying NSTemplateTier resource before modifying it @@ -125,7 +132,7 @@ func UpdateCustomNSTemplateTier(t *testing.T, hostAwait *HostAwaitility, tier *C return tier } -func duplicateTierTemplate(t *testing.T, hostAwait *HostAwaitility, namespace, tierName, origTemplateRef string) (string, error) { +func duplicateTierTemplate(t *testing.T, hostAwait *HostAwaitility, namespace, tierName, origTemplateRef string, modifiers ...TierTemplateModifier) (string, error) { origTierTemplate := &toolchainv1alpha1.TierTemplate{} if err := hostAwait.Client.Get(context.TODO(), test.NamespacedName(hostAwait.Namespace, origTemplateRef), origTierTemplate); err != nil { return "", err @@ -138,6 +145,11 @@ func duplicateTierTemplate(t *testing.T, hostAwait *HostAwaitility, namespace, t }, Spec: origTierTemplate.Spec, } + for _, modify := range modifiers { + err := modify(newTierTemplate) + require.NoError(t, err) + } + newTierTemplate.Spec.TierName = tierName if err := hostAwait.CreateWithCleanup(t, newTierTemplate); err != nil { if !errors.IsAlreadyExists(err) { diff --git a/testsupport/wait/awaitility.go b/testsupport/wait/awaitility.go index 2c3d80b9a..1294c59d5 100644 --- a/testsupport/wait/awaitility.go +++ b/testsupport/wait/awaitility.go @@ -796,6 +796,41 @@ func (w *Waiter[T]) WithNameThat(name string, predicates ...assertions.Predicate return returnedObject, err } +// WithNameDeleted waits for a single object with the provided name in the namespace of the awaitility to get deleted +func (w *Waiter[T]) WithNameDeleted(name string) error { + w.t.Logf("waiting for object of GVK '%s' with name '%s' in namespace '%s' to be deleted", w.gvk, name, w.await.Namespace) + err := wait.Poll(w.await.RetryInterval, w.await.Timeout, func() (done bool, err error) { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(w.gvk) + if err := w.await.Client.Get(context.TODO(), client.ObjectKey{Name: name, Namespace: w.await.Namespace}, obj); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + return false, nil + }) + if err != nil { + sb := strings.Builder{} + sb.WriteString("failed to wait for the the object (GVK '%s') called '%s' in namespace '%s' to be deleted") + args := []any{w.gvk, name, w.await.Namespace} + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(w.gvk) + if err := w.await.Client.Get(context.TODO(), client.ObjectKey{Name: name, Namespace: w.await.Namespace}, obj); err != nil { + sb.WriteString(" and also failed to retrieve the object at all with error: %s") + args = append(args, err) + } else { + o, _ := w.cast(obj) + sb.WriteString(" and the object exists in the cluster:") + content, _ := StringifyObject(o) + sb.WriteRune('\n') + sb.Write(content) + } + w.t.Logf(sb.String(), args...) + } + return err +} + func (w *Waiter[T]) cast(obj *unstructured.Unstructured) (T, error) { var empty T raw, err := obj.MarshalJSON()