Skip to content

Commit

Permalink
Merge pull request #42 from adrianchiris/add-e2e-tests
Browse files Browse the repository at this point in the history
Add E2E tests
  • Loading branch information
adrianchiris authored Nov 3, 2024
2 parents 6ec5a41 + 2e4abf3 commit 821f64d
Show file tree
Hide file tree
Showing 17 changed files with 808 additions and 258 deletions.
25 changes: 25 additions & 0 deletions .github/workflows/build-test-lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -84,3 +84,28 @@ jobs:
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
file: cover.out
e2e-test:
runs-on: ubuntu-latest
needs: [test, build-image]
steps:
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: 1.22.x
- name: Check out code into the Go module directory
uses: actions/checkout@v4
- name: prepare test k8s cluster
run: make test-env-e2e
- name: build and deploy operator
run: make deploy-operator-e2e
- name: run e2e tests
run: make test-e2e
- name: upload artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: e2e-test-artifacts
path: ./artifacts
- name: cleanup
if: always()
run: make clean-test-env-e2e
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -25,5 +25,8 @@ go.work.sum
build/
bin/

# artifact dirs
artifacts/

# IDE
.vscode
88 changes: 67 additions & 21 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ endif
OPERATOR_SDK_VERSION ?= v1.35.0

# Image URL to use all building/pushing image targets
IMG ?= $(IMAGE_TAG_BASE):latest
TAG ?= latest
IMG ?= $(IMAGE_TAG_BASE):$(TAG)
# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
ENVTEST_K8S_VERSION = 1.30.0

Expand Down Expand Up @@ -143,8 +144,7 @@ ifeq (, $(shell which operator-sdk 2>/dev/null))
@{ \
set -e ;\
mkdir -p $(dir $(OPERATOR_SDK)) ;\
OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \
curl -sSLo $(OPERATOR_SDK) https://github.com/operator-framework/operator-sdk/releases/download/$(OPERATOR_SDK_VERSION)/operator-sdk_$${OS}_$${ARCH} ;\
curl -sSLo $(OPERATOR_SDK) https://github.com/operator-framework/operator-sdk/releases/download/$(OPERATOR_SDK_VERSION)/operator-sdk_$(TARGETOS)_$(TARGETARCH) ;\
chmod +x $(OPERATOR_SDK) ;\
}
else
Expand All @@ -160,8 +160,7 @@ ifeq (,$(shell which opm 2>/dev/null))
@{ \
set -e ;\
mkdir -p $(dir $(OPM)) ;\
OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \
curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.23.0/$${OS}-$${ARCH}-opm ;\
curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.23.0/$(TARGETOS)-$(TARGETARCH)-opm ;\
chmod +x $(OPM) ;\
}
else
Expand All @@ -176,8 +175,7 @@ skaffold: $(SKAFFOLD) ## Download skaffold locally if necessary.
$(SKAFFOLD): | $(LOCALBIN)
@{ \
set -e;\
OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \
curl -fsSL https://storage.googleapis.com/skaffold/releases/$(SKAFFOLD_VER)/skaffold-$${OS}-$${ARCH} -o $(SKAFFOLD); \
curl -fsSL https://storage.googleapis.com/skaffold/releases/$(SKAFFOLD_VER)/skaffold-$(TARGETOS)-$(TARGETARCH) -o $(SKAFFOLD); \
chmod +x $(SKAFFOLD);\
}

Expand All @@ -189,11 +187,33 @@ minikube: $(MINIKUBE) ## Download minikube locally if necessary.
$(MINIKUBE): | $(LOCALBIN)
@{ \
set -e;\
OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \
curl -fsSL https://storage.googleapis.com/minikube/releases/$(MINIKUBE_VER)/minikube-$${OS}-$${ARCH} -o $(MINIKUBE); \
curl -fsSL https://storage.googleapis.com/minikube/releases/$(MINIKUBE_VER)/minikube-$(TARGETOS)-$(TARGETARCH) -o $(MINIKUBE); \
chmod +x $(MINIKUBE);\
}

# kind is used to set-up local kubernetes cluster for e2e tests.
KIND_VER := v0.24.0
KIND := $(abspath $(LOCALBIN)/kind-$(KIND_VER))
.PHONY: kind ## Download kind locally if necessary.
kind: $(KIND)
$(KIND): | $(LOCALBIN)
@{ \
set -e; \
test -s $(LOCALBIN)/$(KIND) || GOBIN=$(LOCALBIN) go install sigs.k8s.io/kind@$(KIND_VER); \
mv $(LOCALBIN)/kind $(KIND); \
}

KUBECTL_VER := v1.31.0
KUBECTL := $(abspath $(LOCALBIN)/kubectl-$(KUBECTL_VER))
.PHONY: kubectl ## Download kubectl locally if necessary.
kubectl: $(KUBECTL)
$(KUBECTL): | $(LOCALBIN)
@{ \
set -e;\
curl -fsSL https://dl.k8s.io/release/$(KUBECTL_VER)/bin/$(TARGETOS)/$(TARGETARCH)/kubectl -o $(KUBECTL); \
chmod +x $(KUBECTL);\
}

HELM := $(abspath $(LOCALBIN)/helm)
.PHONY: helm
helm: $(HELM) ## Download helm (last release) locally if necessary.
Expand Down Expand Up @@ -276,10 +296,9 @@ test: unit-test lint
unit-test: envtest ## Run unit tests.
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -cover -covermode=$(COVER_MODE) -coverprofile=$(COVER_PROFILE) $(PKGS)

# Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
.PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up.
test-e2e:
go test ./test/e2e/ -v -ginkgo.v
.PHONY: test-e2e
test-e2e: # Run the e2e tests against a k8s instance with maintenance-operator installed.
go test ./test/e2e/ -v -ginkgo.v -e2e.maintenanceOperatorNamespace=maintenance-operator

.PHONY: lint
lint: golangci-lint ## Run golangci-lint linter & yamllint
Expand Down Expand Up @@ -351,25 +370,43 @@ ifndef ignore-not-found
ignore-not-found = false
endif

KUBECTL ?= kubectl

.PHONY: install
install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
install: manifests kustomize kubectl ## Install CRDs into the K8s cluster specified in ~/.kube/config.
$(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f -

.PHONY: uninstall
uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
uninstall: manifests kustomize kubectl ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
$(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -

.PHONY: deploy
deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
deploy: manifests kustomize kubectl ## Deploy controller to the K8s cluster specified in ~/.kube/config.
cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -

.PHONY: undeploy
undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
undeploy: kubectl ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
$(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -

.PHONY: deploy-operator-e2e
deploy-operator-e2e: helm kubectl kind ## Deploy operator to test cluster
@{ \
echo "Building test image"; \
TAG=test make docker-build; \
echo "Upload test image to kind cluster"; \
$(KIND) load docker-image $(IMAGE_TAG_BASE):test --name $(TEST_CLUSTER_NAME); \
echo "deploy operator to kind cluster"; \
$(HELM) upgrade -i --create-namespace -n maintenance-operator \
--set operator.image.repository=$(IMAGE_TAG_BASE) --set operator.image.tag=test --set operator.image.imagePullPolicy=Never \
maintenance-operator $(CURDIR)/deployment/maintenance-operator-chart; \
}

.PHONY: undeploy-operator-e2e
undeploy-operator-e2e: helm ## Undeploy operator from test cluster
@{ \
$(HELM) uninstall -n maintenance-operator maintenance-operator; \
$(KUBECTL) delete ns maintenance-operator; \
}

##@ Build Dependencies

.PHONY: bundle
Expand Down Expand Up @@ -431,17 +468,26 @@ SKAFFOLD_REGISTRY ?= localhost:5000
.PHONY: dev-env
dev-env: | $(MINIKUBE) $(HELM) ## Create minikube cluster for dev and tests
CLUSTER_NAME=$(TEST_CLUSTER_NAME) MINIKUBE_BIN=$(MINIKUBE) $(CURDIR)/hack/scripts/setup_minikube.sh
CLUSTER_NAME=$(TEST_CLUSTER_NAME) MINIKUBE_BIN=$(MINIKUBE) HELM_BIN=$(HELM) $(CURDIR)/hack/scripts/install_deps.sh
CLUSTER_NAME=$(TEST_CLUSTER_NAME) HELM_BIN=$(HELM) $(CURDIR)/hack/scripts/install_deps.sh

.PHONY: dev-env-multinode
dev-env-multinode: | $(MINIKUBE) $(HELM) ## Create minikube cluster for dev and tests
CLUSTER_NAME=$(TEST_CLUSTER_NAME) MINIKUBE_BIN=$(MINIKUBE) NUM_NODES=4 USE_MINIKUBE_DOCKER=false $(CURDIR)/hack/scripts/setup_minikube.sh
CLUSTER_NAME=$(TEST_CLUSTER_NAME) MINIKUBE_BIN=$(MINIKUBE) HELM_BIN=$(HELM) $(CURDIR)/hack/scripts/install_deps.sh
CLUSTER_NAME=$(TEST_CLUSTER_NAME) HELM_BIN=$(HELM) $(CURDIR)/hack/scripts/install_deps.sh

.PHONY: test-env-e2e
test-env-e2e: | $(KIND) $(HELM) $(KUBECTL) ## Create kind cluster for e2e tests and deploys maintenance operator
CLUSTER_NAME=$(TEST_CLUSTER_NAME) KIND_BIN=$(KIND) KUBECTL_BIN=$(KUBECTL) $(CURDIR)/hack/scripts/setup_kind.sh
CLUSTER_NAME=$(TEST_CLUSTER_NAME) HELM_BIN=$(HELM) $(CURDIR)/hack/scripts/install_deps.sh

.PHONY: clean-dev-env
clean-dev-env: $(MINIKUBE) ## Teardown minikube cluster for dev and tests
$(MINIKUBE) delete -p $(TEST_CLUSTER_NAME)

.PHONY: clean-test-env-e2e
clean-test-env-e2e: kind ## Teardown kind cluster for e2e tests
$(KIND) delete cluster --name $(TEST_CLUSTER_NAME)

.PHONY: dev-operator
dev-operator: $(MINIKUBE) $(SKAFFOLD) ## Deploy maintenance operator controller to dev cluster using skaffold
{\
Expand Down
1 change: 1 addition & 0 deletions deployment/maintenance-operator-chart/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ Maintenance Operator Helm Chart
| operator.admissionController.certificates.secretNames.operator | string | `"operator-webhook-cert"` | secret name containing certificates for the operator admission controller |
| operator.admissionController.enable | bool | `true` | enable admission controller of the operator |
| operator.affinity | object | `{"nodeAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"preference":{"matchExpressions":[{"key":"node-role.kubernetes.io/master","operator":"Exists"}]},"weight":1},{"preference":{"matchExpressions":[{"key":"node-role.kubernetes.io/control-plane","operator":"Exists"}]},"weight":1}]}}` | node affinity for the operator |
| operator.image.imagePullPolicy | string | `nil` | image pull policy for the operator image |
| operator.image.repository | string | `"ghcr.io/mellanox/maintenance-operator"` | repository to use for the operator image |
| operator.image.tag | string | `nil` | image tag to use for the operator image |
| operator.nodeSelector | object | `{}` | node selector for the operator |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ spec:
- name: ENABLE_WEBHOOKS
value: {{ quote .Values.operator.admissionController.enable }}
image: {{ .Values.operator.image.repository }}:{{ .Values.operator.image.tag | default .Chart.AppVersion }}
{{- if .Values.operator.image.imagePullPolicy }}
imagePullPolicy: {{ .Values.operator.image.imagePullPolicy }}
{{- end }}
livenessProbe:
httpGet:
path: /healthz
Expand Down
2 changes: 2 additions & 0 deletions deployment/maintenance-operator-chart/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ operator:
repository: ghcr.io/mellanox/maintenance-operator
# -- image tag to use for the operator image
tag: null
# -- image pull policy for the operator image
imagePullPolicy: null
# -- toleration for the operator
tolerations:
- key: "node-role.kubernetes.io/master"
Expand Down
2 changes: 1 addition & 1 deletion docs/api-reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -436,4 +436,4 @@ href="https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#

--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

*Generated with `gen-crd-api-reference-docs` on git commit `ab32138`.*
*Generated with `gen-crd-api-reference-docs` on git commit `fb44536`.*
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ require (
k8s.io/kubectl v0.31.2
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
sigs.k8s.io/controller-runtime v0.19.1
sigs.k8s.io/yaml v1.4.0
)

require (
Expand Down Expand Up @@ -106,5 +107,4 @@ require (
sigs.k8s.io/kustomize/api v0.17.2 // indirect
sigs.k8s.io/kustomize/kyaml v0.17.1 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
)
8 changes: 8 additions & 0 deletions hack/scripts/e2e-cluster.yaml.template
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
name: ${CLUSTER_NAME}
nodes:
- role: control-plane
- role: worker
- role: worker
- role: worker
15 changes: 1 addition & 14 deletions hack/scripts/install_deps.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,6 @@ if [[ "${TRACE-0}" == "1" ]]; then
set -o xtrace
fi

CLUSTER_NAME=${CLUSTER_NAME:-"mn-op"}
MINIKUBE_BIN=${MINIKUBE_BIN:-"unknown"}
HELM_BIN=${HELM_BIN:-"unknown"}

CERT_MANAGER_VERSION=${CERT_MANAGER_VERSION:-"v1.15.3"}
Expand All @@ -33,23 +31,12 @@ function helm() {
${HELM_BIN} $@
}

function minikube() {
${MINIKUBE_BIN} $@
}

# Check for mandatory vars
if [[ "${MINIKUBE_BIN}" == "unknown" ]]; then
echo "MINIKUBE_BIN not provided. Aborting." >&2
exit 1
fi

if [[ "${HELM_BIN}" == "unknown" ]]; then
echo "HELM_BIN not provided. Aborting." >&2
exit 1
fi

# set minikube profile
minikube profile ${CLUSTER_NAME}
# NOTE: we assume current kubeconfig in home dir points to the test cluster
# install helm
echo "Installing cert-manager ${CERT_MANAGER_VERSION}"
helm repo add jetstack https://charts.jetstack.io --force-update
Expand Down
53 changes: 53 additions & 0 deletions hack/scripts/setup_kind.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
#!/bin/bash
# 2024 NVIDIA CORPORATION & AFFILIATES
#
# Licensed under the Apache License, Version 2.0 (the License);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an AS IS BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o nounset
set -o pipefail
set -o errexit

if [[ "${TRACE-0}" == "1" ]]; then
set -o xtrace
fi

CLUSTER_NAME="${CLUSTER_NAME:-"mn-op"}"
SCRIPT_DIR="$(dirname "$(realpath "$0")")"
KIND_BIN=${KIND_BIN:-"unknown"}
KUBECTL_BIN=${KUBECTL_BIN:-"kubectl"}
CERT_MANAGER_VERSION=${CERT_MANAGER_VERSION:-"v1.15.3"}
CERT_MANAGER_NAMESPACE=${CERT_MANAGER_NAMESPACE:-"cert-manager"}

function kind() {
${KIND_BIN} $@
}

function kubectl() {
${KUBECTL_BIN} $@
}

function label_worker_nodes() {
# label worker nodes as workers. we do this here since doing it in kind config file does not work.
kubectl label nodes --all node-role.kubernetes.io/worker=
kubectl label node ${CLUSTER_NAME}-control-plane node-role.kubernetes.io/worker-
}

if [[ "${KIND_BIN}" == "unknown" ]]; then
echo "KIND_BIN not provided. Aborting." >&2
exit 1
fi

echo "Creating kind cluster ${CLUSTER_NAME}"
CLUSTER_NAME=${CLUSTER_NAME} envsubst < ${SCRIPT_DIR}/e2e-cluster.yaml.template > ${SCRIPT_DIR}/e2e-cluster.yaml
kind create cluster --config ${SCRIPT_DIR}/e2e-cluster.yaml
rm -f ${SCRIPT_DIR}/e2e-cluster.yaml
label_worker_nodes
Loading

0 comments on commit 821f64d

Please sign in to comment.