Skip to content

Commit

Permalink
Add E2E tests
Browse files Browse the repository at this point in the history
add E2E tests for maintenance operator.

Signed-off-by: adrianc <[email protected]>
  • Loading branch information
adrianchiris committed Oct 30, 2024
1 parent 6ec5a41 commit d0fe83a
Show file tree
Hide file tree
Showing 5 changed files with 303 additions and 224 deletions.
13 changes: 10 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,13 @@ $(MINIKUBE): | $(LOCALBIN)
chmod +x $(MINIKUBE);\
}

# kind is used to set-up local kubernetes cluster for e2e tests.
KIND_VER := v0.24.0
KIND := $(abspath $(LOCALBIN)/kind-$(KIND_VER))
.PHONY: kind ## Download kind locally if necessary.
@ test -s $(LOCALBIN)/$(KIND) || GOBIN=$(LOCALBIN) go install sigs.k8s.io/kind$(KIND_VER)


HELM := $(abspath $(LOCALBIN)/helm)
.PHONY: helm
helm: $(HELM) ## Download helm (last release) locally if necessary.
Expand Down Expand Up @@ -276,10 +283,10 @@ test: unit-test lint
unit-test: envtest ## Run unit tests.
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -cover -covermode=$(COVER_MODE) -coverprofile=$(COVER_PROFILE) $(PKGS)

# Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
.PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up.

.PHONY: test-e2e # Run the e2e tests against a k8s instance with maintenance-operator installed.
test-e2e:
go test ./test/e2e/ -v -ginkgo.v
go test ./test/e2e/ -v -ginkgo.v -e2e.maintenanceOperatorNamespace=maintenance-operator

.PHONY: lint
lint: golangci-lint ## Run golangci-lint linter & yamllint
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ require (
k8s.io/kubectl v0.31.2
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
sigs.k8s.io/controller-runtime v0.19.1
sigs.k8s.io/yaml v1.4.0
)

require (
Expand Down Expand Up @@ -106,5 +107,4 @@ require (
sigs.k8s.io/kustomize/api v0.17.2 // indirect
sigs.k8s.io/kustomize/kyaml v0.17.1 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
)
124 changes: 114 additions & 10 deletions test/e2e/e2e_suite_test.go
Original file line number Diff line number Diff line change
@@ -1,32 +1,136 @@
/*
Copyright 2024.
Copyright 2024, NVIDIA CORPORATION & AFFILIATES
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
"context"
"flag"
"fmt"
"os"
"path/filepath"
"testing"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/clientcmd"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log/zap"

maintenancev1 "github.com/Mellanox/maintenance-operator/api/v1alpha1"
)

type testClusterInfo struct {
workerNodes []string
controlPlaneNodes []string
}

// discoverNodes discovers control plane and worker nodes in the cluster.
// then stores them in testClusterInfo
func (tc *testClusterInfo) discoverNodes(k8sClient client.Client) error {
nodes := &corev1.NodeList{}
if err := k8sClient.List(testContext, nodes); err != nil {
return errors.Wrap(err, "failed to list nodes")
}

for _, node := range nodes.Items {
if _, ok := node.Labels["node-role.kubernetes.io/control-plane"]; ok {
tc.controlPlaneNodes = append(tc.controlPlaneNodes, node.Name)
}
if _, ok := node.Labels["node-role.kubernetes.io/worker"]; ok {
tc.workerNodes = append(tc.workerNodes, node.Name)
}
}

if len(tc.controlPlaneNodes) == 0 {
return fmt.Errorf("no control plane nodes found")
}

if len(tc.workerNodes) == 0 {
return fmt.Errorf("no worker nodes found")
}

return nil
}

var (
// testContext is a context for testing.
testContext context.Context
// k8sClient is a Kubernetes client.
k8sClient client.Client
// testKubeconfig is the path to the kubeconfig file used for testing.
testKubeconfig string
// testCluster contains cluster information
testCluster testClusterInfo
// maintenanceOperatorNamespace is the namespace where the maintenance operator is installed.
maintenanceOperatorNamespace string
)

func init() {
flag.StringVar(&testKubeconfig, "e2e.kubeconfig", "", "path to the kubeconfig file used for testing")
flag.StringVar(&maintenanceOperatorNamespace, "e2e.maintenanceOperatorNamespace", "maintenance-operator", "namespace where the maintenance operator is installed")
}

// Run e2e tests using the Ginkgo runner.
func TestE2E(t *testing.T) {
RegisterFailHandler(Fail)
fmt.Fprintf(GinkgoWriter, "Starting maintenance-operator suite\n")
RunSpecs(t, "e2e suite")
}

var _ = BeforeSuite(func() {
fmt.Fprintf(GinkgoWriter, "BeforeSuite\n")
// set up context
testContext = ctrl.SetupSignalHandler()

// set up logger
ctrl.SetLogger(zap.New(zap.UseDevMode(true)))

// setup scheme
fmt.Fprintf(GinkgoWriter, "setup scheme\n")
s := runtime.NewScheme()
Expect(scheme.AddToScheme(s)).To(Succeed())
Expect(maintenancev1.AddToScheme(s)).To(Succeed())

// if testKubeconfig is not set, default it to $HOME/.kube/config
fmt.Fprintf(GinkgoWriter, "get kubeconfig\n")
home, exists := os.LookupEnv("HOME")
Expect(exists).To(BeTrue())
if testKubeconfig == "" {
testKubeconfig = filepath.Join(home, ".kube/config")
}

// create k8sClient
fmt.Fprintf(GinkgoWriter, "create client\n")
restConfig, err := clientcmd.BuildConfigFromFlags("", testKubeconfig)
Expect(err).NotTo(HaveOccurred())
k8sClient, err = client.New(restConfig, client.Options{Scheme: s})
Expect(err).NotTo(HaveOccurred())

// discover nodes
fmt.Fprintf(GinkgoWriter, "discover nodes\n")
Expect(testCluster.discoverNodes(k8sClient)).To(Succeed())

fmt.Fprintf(GinkgoWriter, "Cluster Information\n")
fmt.Fprintf(GinkgoWriter, "ControlPlane Nodes: %+v\n", testCluster.controlPlaneNodes)
fmt.Fprintf(GinkgoWriter, "Worker Nodes: %+v\n", testCluster.workerNodes)
fmt.Fprintf(GinkgoWriter, "BeforeSuite End\n")
})
Loading

0 comments on commit d0fe83a

Please sign in to comment.