Skip to content

Commit

Permalink
Tests for feature toggles (codeready-toolchain#1008)
Browse files Browse the repository at this point in the history
* Tests for feature toggles
---------

Co-authored-by: Matous Jobanek <[email protected]>
Co-authored-by: Rajiv Senthilnathan <[email protected]>
  • Loading branch information
3 people authored Jul 25, 2024
1 parent 82dc979 commit 20debb4
Show file tree
Hide file tree
Showing 6 changed files with 189 additions and 13 deletions.
3 changes: 3 additions & 0 deletions deploy/host-operator/e2e-tests/toolchainconfig.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ spec:
spaceBindingRequestEnabled: true
tiers:
durationBeforeChangeTierRequestDeletion: '5s'
featureToggles:
- name: "test-feature"
weight: 100
toolchainStatus:
toolchainStatusRefreshTime: '1s'
members:
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
module github.com/codeready-toolchain/toolchain-e2e

require (
github.com/codeready-toolchain/api v0.0.0-20240708122235-0af5a9a178bb
github.com/codeready-toolchain/api v0.0.0-20240717145630-bb67a632867a
github.com/codeready-toolchain/toolchain-common v0.0.0-20240716065433-8604fe46b96a
github.com/davecgh/go-spew v1.1.1
github.com/fatih/color v1.12.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,8 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z
github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
github.com/codeready-toolchain/api v0.0.0-20240708122235-0af5a9a178bb h1:Wc9CMsv0ODZv9dM5qF3OI0mFDO95YNIXV/8oRvoz8aE=
github.com/codeready-toolchain/api v0.0.0-20240708122235-0af5a9a178bb/go.mod h1:ie9p4LenCCS0LsnbWp6/xwpFDdCWYE0KWzUO6Sk1g0E=
github.com/codeready-toolchain/api v0.0.0-20240717145630-bb67a632867a h1:La7GOCysmkU+4vnN8lDzXFJwJiA1LWZ9YkX/yQXYnpw=
github.com/codeready-toolchain/api v0.0.0-20240717145630-bb67a632867a/go.mod h1:ie9p4LenCCS0LsnbWp6/xwpFDdCWYE0KWzUO6Sk1g0E=
github.com/codeready-toolchain/toolchain-common v0.0.0-20240716065433-8604fe46b96a h1:HcaJtZCLfYkWZCxIa3iTvq3zgn711JGqPLkunBTfGSc=
github.com/codeready-toolchain/toolchain-common v0.0.0-20240716065433-8604fe46b96a/go.mod h1:8M9k7w2VSyRKSK6P08Jo2ddW3uyGgxCcSitnYa3HK9o=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
Expand Down
126 changes: 126 additions & 0 deletions test/e2e/parallel/nstemplatetier_test.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
package parallel

import (
"bytes"
"context"
"fmt"
"html/template"
v1 "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/runtime"
"testing"
"time"

Expand Down Expand Up @@ -303,3 +307,125 @@ func TestKsctlGeneratedTiers(t *testing.T) {
})
}
}

func TestFeatureToggles(t *testing.T) {
t.Parallel()
awaitilities := WaitForDeployments(t)
hostAwait := awaitilities.Host()
memberAwait := awaitilities.Member1()

base1nsTier, err := hostAwait.WaitForNSTemplateTier(t, "base1ns")
require.NoError(t, err)

t.Run("provision space with enabled feature", func(t *testing.T) {
// given

// Create a new tier which is a copy of base1ns but with an additional ClusterRoleBinding object with "test-feature" annotation.
// "feature-test" feature is defined in the ToolchainConfig and has 100 weight
tier := tiers.CreateCustomNSTemplateTier(t, hostAwait, "ftier", base1nsTier,
withClusterRoleBinding(t, base1nsTier, "test-feature"),
tiers.WithNamespaceResources(t, base1nsTier),
tiers.WithSpaceRoles(t, base1nsTier))
_, err := hostAwait.WaitForNSTemplateTier(t, tier.Name)
require.NoError(t, err)

// when

// Now let's create a Space
user := NewSignupRequest(awaitilities).
Username("featured-user").
Email("[email protected]").
ManuallyApprove().
EnsureMUR().
SpaceTier("base1ns").
TargetCluster(memberAwait).
RequireConditions(wait.ConditionSet(wait.Default(), wait.ApprovedByAdmin())...).
Execute(t)
// and promote that space to the ftier tier
tiers.MoveSpaceToTier(t, hostAwait, "featured-user", tier.Name)
VerifyResourcesProvisionedForSpaceWithCustomTier(t, hostAwait, memberAwait, "featured-user", tier)

// then

// Verify that the space has the feature annotation - the weight is set to 100, so it should be added to all Spaces in all tiers
space, err := hostAwait.WaitForSpace(t, user.Space.Name)
require.NoError(t, err)
require.NotEmpty(t, space.Annotations)
assert.Equal(t, "test-feature", space.Annotations[toolchainv1alpha1.FeatureToggleNameAnnotationKey])
// and CRB for the that feature has been created
crbName := fmt.Sprintf("%s-%s", user.Space.Name, "test-feature")
_, err = wait.For(t, memberAwait.Awaitility, &v1.ClusterRoleBinding{}).WithNameThat(crbName)
require.NoError(t, err)

t.Run("disable feature", func(t *testing.T) {
// when

// Now let's disable the feature for the Space by removing the feature annotation
_, err := hostAwait.UpdateSpace(t, user.Space.Name, func(s *toolchainv1alpha1.Space) {
delete(s.Annotations, toolchainv1alpha1.FeatureToggleNameAnnotationKey)
})
require.NoError(t, err)

// then
err = wait.For(t, memberAwait.Awaitility, &v1.ClusterRoleBinding{}).WithNameDeleted(crbName)
require.NoError(t, err)

t.Run("re-enable feature", func(t *testing.T) {
// when

// Now let's re-enable the feature for the Space by restoring the feature annotation
_, err := hostAwait.UpdateSpace(t, user.Space.Name, func(s *toolchainv1alpha1.Space) {
if s.Annotations == nil {
s.Annotations = make(map[string]string)
}
s.Annotations[toolchainv1alpha1.FeatureToggleNameAnnotationKey] = "test-feature"
})
require.NoError(t, err)

// then
// Verify that the CRB is back
_, err = wait.For(t, memberAwait.Awaitility, &v1.ClusterRoleBinding{}).WithNameThat(crbName)
require.NoError(t, err)
})
})
})
}

func withClusterRoleBinding(t *testing.T, otherTier *toolchainv1alpha1.NSTemplateTier, feature string) tiers.CustomNSTemplateTierModifier {
var tpl bytes.Buffer
err := template.Must(template.New("crb").Parse(viewCRB)).Execute(&tpl, map[string]interface{}{
"featureName": feature,
})
require.NoError(t, err)

return tiers.WithClusterResources(t, otherTier, func(template *toolchainv1alpha1.TierTemplate) error {
clusterRB := runtime.RawExtension{
Raw: tpl.Bytes(),
}
template.Spec.Template.Objects = append(template.Spec.Template.Objects, clusterRB)
return nil
})
}

var viewCRB = `{
"apiVersion": "rbac.authorization.k8s.io/v1",
"kind": "ClusterRoleBinding",
"metadata": {
"name": "${SPACE_NAME}-{{ .featureName }}",
"annotations": {
"toolchain.dev.openshift.com/feature": "{{ .featureName }}"
}
},
"roleRef": {
"apiGroup": "rbac.authorization.k8s.io",
"kind": "ClusterRole",
"name": "view"
},
"subjects": [
{
"kind": "User",
"name": "${USERNAME}"
}
]
}
`
32 changes: 22 additions & 10 deletions testsupport/tiers/tier_setup.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,13 @@ type CustomNSTemplateTier struct {

type CustomNSTemplateTierModifier func(*HostAwaitility, *CustomNSTemplateTier) error

func WithClusterResources(t *testing.T, otherTier *toolchainv1alpha1.NSTemplateTier) CustomNSTemplateTierModifier {
type TierTemplateModifier func(*toolchainv1alpha1.TierTemplate) error

func WithClusterResources(t *testing.T, otherTier *toolchainv1alpha1.NSTemplateTier, modifiers ...TierTemplateModifier) CustomNSTemplateTierModifier {
return func(hostAwait *HostAwaitility, tier *CustomNSTemplateTier) error {
tier.ClusterResourcesTier = otherTier
// configure the "wrapped" NSTemplateTier
tmplRef, err := duplicateTierTemplate(t, hostAwait, otherTier.Namespace, tier.Name, otherTier.Spec.ClusterResources.TemplateRef)
tmplRef, err := duplicateTierTemplate(t, hostAwait, otherTier.Namespace, tier.Name, otherTier.Spec.ClusterResources.TemplateRef, modifiers...)
if err != nil {
return err
}
Expand Down Expand Up @@ -78,6 +80,9 @@ func WithSpaceRoles(t *testing.T, otherTier *toolchainv1alpha1.NSTemplateTier) C
}
}

// CreateCustomNSTemplateTier creates a custom tier.
// If no modifiers provided then the new tier will use copies of the baseTier cluster, namespace and space roles templates
// without any modifications.
func CreateCustomNSTemplateTier(t *testing.T, hostAwait *HostAwaitility, name string, baseTier *toolchainv1alpha1.NSTemplateTier, modifiers ...CustomNSTemplateTierModifier) *CustomNSTemplateTier {
tier := &CustomNSTemplateTier{
NSTemplateTier: &toolchainv1alpha1.NSTemplateTier{
Expand All @@ -91,12 +96,14 @@ func CreateCustomNSTemplateTier(t *testing.T, hostAwait *HostAwaitility, name st
},
},
}
// add default values before custom values...
modifiers = append([]CustomNSTemplateTierModifier{
WithClusterResources(t, baseTier),
WithNamespaceResources(t, baseTier),
WithSpaceRoles(t, baseTier),
}, modifiers...)
if len(modifiers) == 0 {
// If no modifiers provided then use default modifiers which would use resources from the base tier.
modifiers = []CustomNSTemplateTierModifier{
WithClusterResources(t, baseTier),
WithNamespaceResources(t, baseTier),
WithSpaceRoles(t, baseTier),
}
}

// ... and apply
for _, modify := range modifiers {
Expand All @@ -108,7 +115,7 @@ func CreateCustomNSTemplateTier(t *testing.T, hostAwait *HostAwaitility, name st
return tier
}

// createCustomNSTemplateTier updates the given "tier" using the modifiers
// UpdateCustomNSTemplateTier updates the given "tier" using the modifiers
// returns the latest version of the NSTemplateTier
func UpdateCustomNSTemplateTier(t *testing.T, hostAwait *HostAwaitility, tier *CustomNSTemplateTier, modifiers ...CustomNSTemplateTierModifier) *CustomNSTemplateTier {
// reload the underlying NSTemplateTier resource before modifying it
Expand All @@ -125,7 +132,7 @@ func UpdateCustomNSTemplateTier(t *testing.T, hostAwait *HostAwaitility, tier *C
return tier
}

func duplicateTierTemplate(t *testing.T, hostAwait *HostAwaitility, namespace, tierName, origTemplateRef string) (string, error) {
func duplicateTierTemplate(t *testing.T, hostAwait *HostAwaitility, namespace, tierName, origTemplateRef string, modifiers ...TierTemplateModifier) (string, error) {
origTierTemplate := &toolchainv1alpha1.TierTemplate{}
if err := hostAwait.Client.Get(context.TODO(), test.NamespacedName(hostAwait.Namespace, origTemplateRef), origTierTemplate); err != nil {
return "", err
Expand All @@ -138,6 +145,11 @@ func duplicateTierTemplate(t *testing.T, hostAwait *HostAwaitility, namespace, t
},
Spec: origTierTemplate.Spec,
}
for _, modify := range modifiers {
err := modify(newTierTemplate)
require.NoError(t, err)
}

newTierTemplate.Spec.TierName = tierName
if err := hostAwait.CreateWithCleanup(t, newTierTemplate); err != nil {
if !errors.IsAlreadyExists(err) {
Expand Down
35 changes: 35 additions & 0 deletions testsupport/wait/awaitility.go
Original file line number Diff line number Diff line change
Expand Up @@ -796,6 +796,41 @@ func (w *Waiter[T]) WithNameThat(name string, predicates ...assertions.Predicate
return returnedObject, err
}

// WithNameDeleted waits for a single object with the provided name in the namespace of the awaitility to get deleted
func (w *Waiter[T]) WithNameDeleted(name string) error {
w.t.Logf("waiting for object of GVK '%s' with name '%s' in namespace '%s' to be deleted", w.gvk, name, w.await.Namespace)
err := wait.Poll(w.await.RetryInterval, w.await.Timeout, func() (done bool, err error) {
obj := &unstructured.Unstructured{}
obj.SetGroupVersionKind(w.gvk)
if err := w.await.Client.Get(context.TODO(), client.ObjectKey{Name: name, Namespace: w.await.Namespace}, obj); err != nil {
if apierrors.IsNotFound(err) {
return true, nil
}
return false, err
}
return false, nil
})
if err != nil {
sb := strings.Builder{}
sb.WriteString("failed to wait for the the object (GVK '%s') called '%s' in namespace '%s' to be deleted")
args := []any{w.gvk, name, w.await.Namespace}
obj := &unstructured.Unstructured{}
obj.SetGroupVersionKind(w.gvk)
if err := w.await.Client.Get(context.TODO(), client.ObjectKey{Name: name, Namespace: w.await.Namespace}, obj); err != nil {
sb.WriteString(" and also failed to retrieve the object at all with error: %s")
args = append(args, err)
} else {
o, _ := w.cast(obj)
sb.WriteString(" and the object exists in the cluster:")
content, _ := StringifyObject(o)
sb.WriteRune('\n')
sb.Write(content)
}
w.t.Logf(sb.String(), args...)
}
return err
}

func (w *Waiter[T]) cast(obj *unstructured.Unstructured) (T, error) {
var empty T
raw, err := obj.MarshalJSON()
Expand Down

0 comments on commit 20debb4

Please sign in to comment.