diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 000000000..2f526620a
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,8 @@
+.github/
+.idea/
+bin/
+hack/
+charts/
+deploy/
+docs/
+examples/
diff --git a/.github/workflows/publish-ecr.yaml b/.github/workflows/publish-ecr.yaml
index 1c32b44ee..e6a326743 100644
--- a/.github/workflows/publish-ecr.yaml
+++ b/.github/workflows/publish-ecr.yaml
@@ -39,7 +39,7 @@ jobs:
           echo "VERSION=${GITHUB_REF_NAME}" >> $GITHUB_ENV
 
     - name: Build, tag, and push manifest to Amazon ECR
-      run: make -j `nproc` all-push
+      run: make -j `nproc` all-push-with-a1compat
 
   ecr-public:
     name: Push to ECR Public
diff --git a/.gitignore b/.gitignore
index 000ff70b0..4be608e59 100644
--- a/.gitignore
+++ b/.gitignore
@@ -33,3 +33,7 @@ vendor/
 
 # Files used by Makefile when upgrading sidecars
 hack/release-scripts/image-digests.yaml
+
+# E2E artifacts
+_rundir/
+_artifacts/
diff --git a/Dockerfile b/Dockerfile
index 5ea6b1b04..a60b4be99 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -24,7 +24,7 @@ COPY . .
 ARG TARGETOS
 ARG TARGETARCH
 ARG VERSION
-RUN OS=$TARGETOS ARCH=$TARGETARCH make $TARGETOS/$TARGETARCH
+RUN OS=$TARGETOS ARCH=$TARGETARCH make
 
 FROM public.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base-csi-ebs:latest-al23 AS linux-al2023
 COPY --from=builder /go/src/github.com/kubernetes-sigs/aws-ebs-csi-driver/bin/aws-ebs-csi-driver /bin/aws-ebs-csi-driver
diff --git a/Makefile b/Makefile
index 697aa08c3..726244f70 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-# Copyright 2019 The Kubernetes Authors.
+# Copyright 2023 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,29 +12,35 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+###
+### How to use this Makefile is documented in docs/makefile.md
+###
+
+## Variables/Functions
+
 VERSION?=v1.25.0
 
 PKG=github.com/kubernetes-sigs/aws-ebs-csi-driver
 GIT_COMMIT?=$(shell git rev-parse HEAD)
 BUILD_DATE?=$(shell date -u -Iseconds)
-
 LDFLAGS?="-X ${PKG}/pkg/driver.driverVersion=${VERSION} -X ${PKG}/pkg/cloud.driverVersion=${VERSION} -X ${PKG}/pkg/driver.gitCommit=${GIT_COMMIT} -X ${PKG}/pkg/driver.buildDate=${BUILD_DATE} -s -w"
 
-GO111MODULE=on
-GOPATH=$(shell go env GOPATH)
-GOOS=$(shell go env GOOS)
-GOBIN=$(shell pwd)/bin
+OS?=$(shell go env GOHOSTOS)
+ARCH?=$(shell go env GOHOSTARCH)
+ifeq ($(OS),windows)
+	BINARY=aws-ebs-csi-driver.exe
+	OSVERSION?=ltsc2022
+else
+	BINARY=aws-ebs-csi-driver
+	OSVERSION?=al2023
+endif
+
+GO_SOURCES=go.mod go.sum $(shell find pkg cmd -type f -name "*.go")
 
 REGISTRY?=gcr.io/k8s-staging-provider-aws
 IMAGE?=$(REGISTRY)/aws-ebs-csi-driver
 TAG?=$(GIT_COMMIT)
 
-OUTPUT_TYPE?=docker
-
-OS?=linux
-ARCH?=amd64
-OSVERSION?=al2023
-
 ALL_OS?=linux windows
 ALL_ARCH_linux?=amd64 arm64
 ALL_OSVERSION_linux?=al2023
@@ -43,245 +49,108 @@ ALL_OS_ARCH_OSVERSION_linux=$(foreach arch, $(ALL_ARCH_linux), $(foreach osversi
 ALL_ARCH_windows?=amd64
 ALL_OSVERSION_windows?=ltsc2019 ltsc2022
 ALL_OS_ARCH_OSVERSION_windows=$(foreach arch, $(ALL_ARCH_windows), $(foreach osversion, ${ALL_OSVERSION_windows}, windows-$(arch)-${osversion}))
-
 ALL_OS_ARCH_OSVERSION=$(foreach os, $(ALL_OS), ${ALL_OS_ARCH_OSVERSION_${os}})
 
+CLUSTER_NAME?=ebs-csi-e2e.k8s.local
+CLUSTER_TYPE?=kops
+WINDOWS?=false
+
 # split words on hyphen, access by 1-index
 word-hyphen = $(word $2,$(subst -, ,$1))
 
 .EXPORT_ALL_VARIABLES:
 
-.PHONY: linux/$(ARCH) bin/aws-ebs-csi-driver
-linux/$(ARCH): bin/aws-ebs-csi-driver
-bin/aws-ebs-csi-driver: | bin
-	CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) go build -mod=mod -ldflags ${LDFLAGS} -o bin/aws-ebs-csi-driver ./cmd/
-
-.PHONY: windows/$(ARCH) bin/aws-ebs-csi-driver.exe
-windows/$(ARCH): bin/aws-ebs-csi-driver.exe
-bin/aws-ebs-csi-driver.exe: | bin
-	CGO_ENABLED=0 GOOS=windows GOARCH=$(ARCH) go build -mod=mod -ldflags ${LDFLAGS} -o bin/aws-ebs-csi-driver.exe ./cmd/
+## Default target
+# When no target is supplied, make runs the first target that does not begin with a .
+# Alias that to building the binary
+.PHONY: default
+default: bin/$(BINARY)
 
-# Builds all linux images (not windows because it can't be exported with OUTPUT_TYPE=docker)
-.PHONY: all
-all: all-image-docker
-
-# Builds all linux and windows images and pushes them
-.PHONY: all-push
-all-push: all-image-registry push-manifest
-
-.PHONY: push-manifest
-push-manifest: create-manifest
-	docker manifest push --purge $(IMAGE):$(TAG)
-
-.PHONY: create-manifest
-create-manifest: all-image-registry
-# sed expression:
-# LHS: match 0 or more not space characters
-# RHS: replace with $(IMAGE):$(TAG)-& where & is what was matched on LHS
-	docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(ALL_OS_ARCH_OSVERSION) | sed -e "s~[^ ]*~$(IMAGE):$(TAG)\-&~g")
-
-# Only linux for OUTPUT_TYPE=docker because windows image cannot be exported
-# "Currently, multi-platform images cannot be exported with the docker export type. The most common usecase for multi-platform images is to directly push to a registry (see registry)."
-# https://docs.docker.com/engine/reference/commandline/buildx_build/#output
-.PHONY: all-image-docker
-all-image-docker: $(addprefix sub-image-docker-,$(ALL_OS_ARCH_OSVERSION_linux))
-.PHONY: all-image-registry
-all-image-registry: sub-image-registry-linux-arm64-al2 $(addprefix sub-image-registry-,$(ALL_OS_ARCH_OSVERSION))
-
-sub-image-%:
-	$(MAKE) OUTPUT_TYPE=$(call word-hyphen,$*,1) OS=$(call word-hyphen,$*,2) ARCH=$(call word-hyphen,$*,3) OSVERSION=$(call word-hyphen,$*,4) image
-
-.PHONY: image
-image: .image-$(TAG)-$(OS)-$(ARCH)-$(OSVERSION)
-.image-$(TAG)-$(OS)-$(ARCH)-$(OSVERSION):
-	docker buildx build \
-		--platform=$(OS)/$(ARCH) \
-		--progress=plain \
-		--target=$(OS)-$(OSVERSION) \
-		--output=type=$(OUTPUT_TYPE) \
-		-t=$(IMAGE):$(TAG)-$(OS)-$(ARCH)-$(OSVERSION) \
-		--build-arg=GOPROXY=$(GOPROXY) \
-		--build-arg=VERSION=$(VERSION) \
-		`./hack/provenance` \
-		.
-	touch $@
+## Top level targets
+# Targets intended to be executed by humans via `make target`
+# See docs/makefile.md for more information
 
 .PHONY: clean
 clean:
-	rm -rf .*image-* bin/
+	rm -rf bin/
 
-bin /tmp/helm /tmp/kubeval:
-	@mkdir -p $@
+.PHONY: test
+test:
+	go test -v -race ./cmd/... ./pkg/...
 
-bin/helm: | /tmp/helm bin
-	@curl -o /tmp/helm/helm.tar.gz -sSL https://get.helm.sh/helm-v3.11.2-${GOOS}-amd64.tar.gz
-	@tar -zxf /tmp/helm/helm.tar.gz -C bin --strip-components=1
-	@rm -rf /tmp/helm/*
+.PHONY: test-sanity
+test-sanity:
+	echo "succeed"
 
-bin/kubeval: | /tmp/kubeval bin
-	@curl -o /tmp/kubeval/kubeval.tar.gz -sSL https://github.com/instrumenta/kubeval/releases/download/0.16.1/kubeval-linux-amd64.tar.gz
-	@tar -zxf /tmp/kubeval/kubeval.tar.gz -C bin kubeval
-	@rm -rf /tmp/kubeval/*
+.PHONY: tools
+tools: bin/aws bin/ct bin/eksctl bin/ginkgo bin/golangci-lint bin/helm bin/kops bin/kubetest2 bin/mockgen bin/shfmt
 
-bin/mockgen: | bin
-	go install github.com/golang/mock/mockgen@v1.6.0
+.PHONY: update
+update: update/gofmt update/kustomize update/mockgen update/gomod update/shfmt
+	@echo "All updates succeeded!"
 
-bin/golangci-lint: | bin
-	echo "Installing golangci-lint..."
-	curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v1.54.0
+.PHONY: verify
+verify: verify/govet verify/golangci-lint verify/update
+	@echo "All verifications passed!"
 
-.PHONY: kubeval
-kubeval: bin/kubeval
-	bin/kubeval -d deploy/kubernetes/base,deploy/kubernetes/cluster,deploy/kubernetes/overlays -i kustomization.yaml,crd_.+\.yaml,controller_add
+.PHONY: all-push
+all-push: all-image-registry push-manifest
 
-.PHONY: mockgen
-mockgen: bin/mockgen
-	./hack/update-gomock
+.PHONY: cluster/create
+cluster/create: bin/kops bin/eksctl
+	./hack/e2e/create-cluster.sh
 
-.PHONY: verify
-verify: bin/golangci-lint
-	echo "verifying and linting files ..."
-	./hack/verify-all
-	echo "Congratulations! All Go source files have been linted."
+.PHONY: cluster/delete
+cluster/delete: bin/kops bin/eksctl
+	./hack/e2e/delete-cluster.sh
 
-.PHONY: test
-test:
-	go test -v -race ./cmd/... ./pkg/...
+## E2E targets
+# Targets to run e2e tests
 
-.PHONY: test-sanity
-test-sanity:
-	#go test -v ./tests/sanity/...
-	echo "succeed"
-
-.PHONY: test-e2e-single-az
-test-e2e-single-az:
-	AWS_REGION=us-west-2 \
+.PHONY: e2e/single-az
+e2e/single-az: bin/helm bin/ginkgo
 	AWS_AVAILABILITY_ZONES=us-west-2a \
-	HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME,controller.volumeModificationFeature.enabled=true' \
-	EBS_INSTALL_SNAPSHOT="true" \
 	TEST_PATH=./tests/e2e/... \
 	GINKGO_FOCUS="\[ebs-csi-e2e\] \[single-az\]" \
-	GINKGO_SKIP="\"sc1\"|\"st1\"" \
+	HELM_EXTRA_FLAGS="--set=controller.volumeModificationFeature.enabled=true" \
 	./hack/e2e/run.sh
 
-.PHONY: test-e2e-multi-az
-test-e2e-multi-az:
-	AWS_REGION=us-west-2 \
-	AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \
-	HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \
-	EBS_INSTALL_SNAPSHOT="true" \
+.PHONY: e2e/multi-az
+e2e/multi-az: bin/helm bin/ginkgo
 	TEST_PATH=./tests/e2e/... \
 	GINKGO_FOCUS="\[ebs-csi-e2e\] \[multi-az\]" \
 	./hack/e2e/run.sh
 
-.PHONY: test-e2e-external
-test-e2e-external:
-	AWS_REGION=us-west-2 \
-	AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \
-	HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \
-	EBS_INSTALL_SNAPSHOT="true" \
-	TEST_PATH=./tests/e2e-kubernetes/... \
-	GINKGO_FOCUS="External.Storage" \
-	GINKGO_SKIP="\[Disruptive\]|\[Serial\]" \
+.PHONY: e2e/external
+e2e/external: bin/helm bin/kubetest2
 	COLLECT_METRICS="true" \
 	./hack/e2e/run.sh
 
-.PHONY: test-e2e-external-arm64
-test-e2e-external-arm64:
-	AWS_REGION=us-west-2 \
-	AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \
-	HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \
-	EBS_INSTALL_SNAPSHOT="true" \
-	TEST_PATH=./tests/e2e-kubernetes/... \
-	GINKGO_FOCUS="External.Storage" \
-	GINKGO_SKIP="\[Disruptive\]|\[Serial\]" \
-	INSTANCE_TYPE="m7g.medium" \
-	AMI_PARAMETER="/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64" \
+.PHONY: e2e/external-arm64
+e2e/external-arm64: bin/helm bin/kubetest2
 	IMAGE_ARCH="arm64" \
 	./hack/e2e/run.sh
 
-.PHONY: test-e2e-external-eks
-test-e2e-external-eks:
-	CLUSTER_TYPE=eksctl \
-	HELM_VALUES_FILE="./hack/values_eksctl.yaml" \
-	HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \
-	EBS_INSTALL_SNAPSHOT="true" \
-	EKSCTL_ADMIN_ROLE="Infra-prod-KopsDeleteAllLambdaServiceRoleF1578477-1ELDFIB4KCMXV" \
-	AWS_REGION=us-west-2 \
-	AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b \
-	TEST_PATH=./tests/e2e-kubernetes/... \
-	GINKGO_FOCUS="External.Storage" \
-	GINKGO_SKIP="\[Disruptive\]|\[Serial\]" \
-	./hack/e2e/run.sh
-
-.PHONY: test-e2e-external-eks-windows
-test-e2e-external-eks-windows:
-	CLUSTER_TYPE=eksctl \
+.PHONY: e2e/external-windows
+e2e/external-windows: bin/helm bin/kubetest2
 	WINDOWS=true \
-	HELM_VALUES_FILE="./hack/values_eksctl.yaml" \
-	HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \
-	EKSCTL_ADMIN_ROLE="Infra-prod-KopsDeleteAllLambdaServiceRoleF1578477-1ELDFIB4KCMXV" \
-	AWS_REGION=us-west-2 \
-	AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b \
-	TEST_PATH=./tests/e2e-kubernetes/... \
-	GINKGO_FOCUS="External.Storage" \
 	GINKGO_SKIP="\[Disruptive\]|\[Serial\]|\[LinuxOnly\]|\[Feature:VolumeSnapshotDataSource\]|\(xfs\)|\(ext4\)|\(block volmode\)" \
 	GINKGO_PARALLEL=15 \
-	NODE_OS_DISTRO="windows" \
+	EBS_INSTALL_SNAPSHOT="false" \
 	./hack/e2e/run.sh
 
-.PHONY: test-e2e-external-kustomize
-test-e2e-external-kustomize:
-	AWS_REGION=us-west-2 \
-	AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \
-	EBS_INSTALL_SNAPSHOT="true" \
-	TEST_PATH=./tests/e2e-kubernetes/... \
-	GINKGO_FOCUS="External.Storage" \
-	GINKGO_SKIP="\[Disruptive\]|\[Serial\]" \
+.PHONY: e2e/external-kustomize
+e2e/external-kustomize: bin/kubetest2
 	DEPLOY_METHOD="kustomize" \
 	./hack/e2e/run.sh
 
-.PHONY: test-helm-chart
-test-helm-chart:
-	AWS_REGION=us-west-2 \
-	AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \
-	EBS_INSTALL_SNAPSHOT="true" \
+.PHONY: e2e/helm-ct
+e2e/helm-ct: bin/helm bin/ct
 	HELM_CT_TEST="true" \
 	./hack/e2e/run.sh
 
-.PHONY: verify-vendor
-test: verify-vendor
-verify: verify-vendor
-verify-vendor:
-	@ echo; echo "### $@:"
-	@ ./hack/verify-vendor.sh
-
-.PHONY: verify-kustomize
-verify: verify-kustomize
-verify-kustomize:
-	@ echo; echo "### $@:"
-	@ ./hack/verify-kustomize
-
-.PHONY: generate-kustomize
-generate-kustomize: bin/helm
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrole-attacher.yaml > ../../deploy/kubernetes/base/clusterrole-attacher.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrole-csi-node.yaml > ../../deploy/kubernetes/base/clusterrole-csi-node.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrole-provisioner.yaml > ../../deploy/kubernetes/base/clusterrole-provisioner.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrole-resizer.yaml > ../../deploy/kubernetes/base/clusterrole-resizer.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrole-snapshotter.yaml > ../../deploy/kubernetes/base/clusterrole-snapshotter.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrolebinding-attacher.yaml > ../../deploy/kubernetes/base/clusterrolebinding-attacher.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrolebinding-csi-node.yaml > ../../deploy/kubernetes/base/clusterrolebinding-csi-node.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrolebinding-provisioner.yaml > ../../deploy/kubernetes/base/clusterrolebinding-provisioner.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrolebinding-resizer.yaml > ../../deploy/kubernetes/base/clusterrolebinding-resizer.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrolebinding-snapshotter.yaml > ../../deploy/kubernetes/base/clusterrolebinding-snapshotter.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/controller.yaml --api-versions 'snapshot.storage.k8s.io/v1' --set 'controller.userAgentExtra=kustomize' | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/controller.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/csidriver.yaml > ../../deploy/kubernetes/base/csidriver.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/node.yaml | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/node.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/poddisruptionbudget-controller.yaml --api-versions 'policy/v1/PodDisruptionBudget' | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/poddisruptionbudget-controller.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/serviceaccount-csi-controller.yaml | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/serviceaccount-csi-controller.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/serviceaccount-csi-node.yaml | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/serviceaccount-csi-node.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/role-leases.yaml | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/role-leases.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/rolebinding-leases.yaml | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/rolebinding-leases.yaml
+## Release scripts
+# Targets run as part of performing a release
 
 .PHONY: update-truth-sidecars
 update-truth-sidecars: hack/release-scripts/get-latest-sidecar-images
@@ -292,4 +161,101 @@ generate-sidecar-tags: update-truth-sidecars charts/aws-ebs-csi-driver/values.ya
 	./hack/release-scripts/generate-sidecar-tags
 
 .PHONY: update-sidecar-dependencies
-update-sidecar-dependencies: update-truth-sidecars generate-sidecar-tags generate-kustomize
+update-sidecar-dependencies: update-truth-sidecars generate-sidecar-tags update/kustomize
+
+## CI aliases
+# Targets intended to be executed mostly or only by CI jobs
+
+.PHONY: all-push-with-a1compat
+all-push-with-a1compat: sub-image-linux-arm64-al2 all-image-registry push-manifest
+
+test-e2e-%:
+	./hack/prow-e2e.sh test-e2e-$*
+
+test-helm-chart:
+	./hack/prow-e2e.sh test-helm-chart
+
+## Builds
+
+bin/$(BINARY): $(GO_SOURCES) | bin
+	CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -mod=readonly -ldflags ${LDFLAGS} -o $@ ./cmd/
+
+.PHONY: all-image-registry
+all-image-registry: $(addprefix sub-image-,$(ALL_OS_ARCH_OSVERSION))
+
+sub-image-%:
+	$(MAKE) OS=$(call word-hyphen,$*,1) ARCH=$(call word-hyphen,$*,2) OSVERSION=$(call word-hyphen,$*,3) image
+
+.PHONY: image
+image:
+	docker buildx build \
+		--platform=$(OS)/$(ARCH) \
+		--progress=plain \
+		--target=$(OS)-$(OSVERSION) \
+		--output=type=registry \
+		-t=$(IMAGE):$(TAG)-$(OS)-$(ARCH)-$(OSVERSION) \
+		--build-arg=GOPROXY=$(GOPROXY) \
+		--build-arg=VERSION=$(VERSION) \
+		`./hack/provenance.sh` \
+		.
+
+.PHONY: create-manifest
+create-manifest: all-image-registry
+# sed expression:
+# LHS: match 0 or more not space characters
+# RHS: replace with $(IMAGE):$(TAG)-& where & is what was matched on LHS
+	docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(ALL_OS_ARCH_OSVERSION) | sed -e "s~[^ ]*~$(IMAGE):$(TAG)\-&~g")
+
+.PHONY: push-manifest
+push-manifest: create-manifest
+	docker manifest push --purge $(IMAGE):$(TAG)
+
+## Tools
+# Tools necessary to perform other targets
+
+bin/%: hack/tools/install.sh hack/tools/python-runner.sh
+	@TOOLS_PATH="$(shell pwd)/bin" ./hack/tools/install.sh $*
+
+## Updaters
+# Automatic generators/formatters for code
+
+.PHONY: update/gofmt
+update/gofmt:
+	gofmt -s -w .
+
+.PHONY: update/kustomize
+update/kustomize: bin/helm
+	./hack/update-kustomize.sh
+
+.PHONY: update/mockgen
+update/mockgen: bin/mockgen
+	./hack/update-mockgen.sh
+
+.PHONY: update/gomod
+update/gomod:
+	go mod tidy
+
+.PHONY: update/shfmt
+update/shfmt: bin/shfmt
+	./bin/shfmt -w -i 2 -d .
+
+## Verifiers
+# Linters and similar
+
+.PHONY: verify/golangci-lint
+verify/golangci-lint: bin/golangci-lint
+	./bin/golangci-lint run --timeout=10m --verbose
+
+.PHONY: verify/govet
+verify/govet:
+	go vet $$(go list ./...)
+
+.PHONY: verify/update
+verify/update: bin/helm bin/mockgen
+	./hack/verify-update.sh
+
+## Internal
+
+# Directories
+bin:
+	@mkdir -p $@
diff --git a/deploy/kubernetes/base/clusterrolebinding-attacher.yaml b/deploy/kubernetes/base/clusterrolebinding-attacher.yaml
index 5715d2651..12405a09b 100644
--- a/deploy/kubernetes/base/clusterrolebinding-attacher.yaml
+++ b/deploy/kubernetes/base/clusterrolebinding-attacher.yaml
@@ -9,7 +9,6 @@ metadata:
 subjects:
   - kind: ServiceAccount
     name: ebs-csi-controller-sa
-    namespace: default
 roleRef:
   kind: ClusterRole
   name: ebs-external-attacher-role
diff --git a/deploy/kubernetes/base/clusterrolebinding-csi-node.yaml b/deploy/kubernetes/base/clusterrolebinding-csi-node.yaml
index 095db5251..4cc33b7e8 100644
--- a/deploy/kubernetes/base/clusterrolebinding-csi-node.yaml
+++ b/deploy/kubernetes/base/clusterrolebinding-csi-node.yaml
@@ -9,7 +9,6 @@ metadata:
 subjects:
   - kind: ServiceAccount
     name: ebs-csi-node-sa
-    namespace: default
 roleRef:
   kind: ClusterRole
   name: ebs-csi-node-role
diff --git a/deploy/kubernetes/base/clusterrolebinding-provisioner.yaml b/deploy/kubernetes/base/clusterrolebinding-provisioner.yaml
index 3544bc61e..e95a94b96 100644
--- a/deploy/kubernetes/base/clusterrolebinding-provisioner.yaml
+++ b/deploy/kubernetes/base/clusterrolebinding-provisioner.yaml
@@ -9,7 +9,6 @@ metadata:
 subjects:
   - kind: ServiceAccount
     name: ebs-csi-controller-sa
-    namespace: default
 roleRef:
   kind: ClusterRole
   name: ebs-external-provisioner-role
diff --git a/deploy/kubernetes/base/clusterrolebinding-resizer.yaml b/deploy/kubernetes/base/clusterrolebinding-resizer.yaml
index c80a9a26b..543086e74 100644
--- a/deploy/kubernetes/base/clusterrolebinding-resizer.yaml
+++ b/deploy/kubernetes/base/clusterrolebinding-resizer.yaml
@@ -9,7 +9,6 @@ metadata:
 subjects:
   - kind: ServiceAccount
     name: ebs-csi-controller-sa
-    namespace: default
 roleRef:
   kind: ClusterRole
   name: ebs-external-resizer-role
diff --git a/deploy/kubernetes/base/clusterrolebinding-snapshotter.yaml b/deploy/kubernetes/base/clusterrolebinding-snapshotter.yaml
index 7946414d5..81ed7c2b8 100644
--- a/deploy/kubernetes/base/clusterrolebinding-snapshotter.yaml
+++ b/deploy/kubernetes/base/clusterrolebinding-snapshotter.yaml
@@ -9,7 +9,6 @@ metadata:
 subjects:
   - kind: ServiceAccount
     name: ebs-csi-controller-sa
-    namespace: default
 roleRef:
   kind: ClusterRole
   name: ebs-external-snapshotter-role
diff --git a/docs/makefile.md b/docs/makefile.md
new file mode 100644
index 000000000..8f97e0ea0
--- /dev/null
+++ b/docs/makefile.md
@@ -0,0 +1,198 @@
+# Use of the EBS CSI Driver `Makefile`
+
+The EBS CSI Driver comes with a Makefile that can be used to develop, build, test, and release the driver. This file documents the available targets intended for use by humans, the parameters they support, and common usage scenarios.
+
+## Prerequisites
+
+The `Makefile` and associated tooling assume you already have the following tools installed:
+- `go`: https://go.dev/doc/install
+- `python` (may be named `python3`) and `pip`: https://www.python.org/downloads/
+- `jq`: https://github.com/jqlang/jq/releases
+- `kubectl`: https://kubernetes.io/docs/tasks/tools/#kubectl
+- `git`: https://git-scm.com/downloads
+- `docker` and `docker buildx`: https://docs.docker.com/get-docker/ and https://github.com/docker/buildx#installing
+- `make`
+- Standard POSIX tools (`awk`, `grep`, `cat`, etc)
+
+All other tools are downloaded for you at runtime.
+
+## Quickstart Guide
+
+This guide demonstrates the basic workflow for developing for the EBS CSI Driver. More detailed documentation of the available `make` targets is available below.
+
+### 1. Local development for the EBS CSI Driver
+
+If your changes are Helm-only, skip to section 2 (Run E2E tests) below after making your changes.
+
+During development, use `make` at any time to build a driver binary, and thus discover compiler errors. When your change is ready to be tested, run `make test` to execute the unit test suite for the driver. If you are making a significant change to the driver (such as a bugfix or new feature), please add new unit tests or update existing tests where applicable.
+
+### 2. Run E2E tests
+
+To create a `kops` cluster to test the driver against:
+```bash
+make cluster/create
+```
+
+If your change affects the Windows implementation of the driver, instead create an `eksctl` cluster with Windows nodes:
+```bash
+export WINDOWS="true"
+# Note: CLUSTER_TYPE must be set for all e2e tests and cluster deletion
+# Re-export it if necessary (for example, if running tests in a separate terminal tab)
+export CLUSTER_TYPE="eksctl"
+make cluster/create
+```
+
+If you are making a change to the driver, the recommended test suite to run is the external tests:
+```bash
+# Normal external tests (excluding Windows tests)
+make e2e/external
+# Instead, if testing Windows
+make e2e/external-windows
+```
+
+If you are making a change to the Helm chart, the recommended test suite to run is the Helm `ct` tests:
+```bash
+make e2e/helm-ct
+```
+
+To cleanup your cluster after finishing testing:
+```bash
+make cluster/delete
+```
+
+### 3. Before submitting a PR
+
+Run `make update` to automatically format go source files and re-generate automatically generated files. If `make update` produces any changes, commit them before submitting a PR.
+
+Run `make verify` to run linters and other similar code checking tools. Fix any issues `make verify` discovers before submitting a PR.
+
+## Building
+
+### `make` or `make bin/aws-ebs-csi-driver` or `make bin/aws-ebs-csi-driver.exe`
+
+Build a binary copy of the EBS CSI Driver for the local platform. This is the default behavior when calling `make` with no target.
+
+The target OS and/or architecture can be overridden via the `OS` and `ARCH` environment variables (for example, `OS=linux ARCH=arm64 make`)
+
+### `make image`
+
+Build and push a single image of the driver based on the local platform (the same overrides as `make` apply, as well as `OSVERSION` to override container OS version). In most cases, `make all-push` is more suitable. Environment variables are accepted to override the `REGISTRY`, `IMAGE` name, and image `TAG`.
+
+### `make all-push`
+
+Build and push a multi-arch image of the driver based on the OSes in `ALL_OS`, architectures in `ALL_ARCH_linux`/`ALL_ARCH_windows`, and OS versions in `ALL_OSVERSION_linux`/`ALL_OSVERSION_windows`. Also supports `REGISTRY`, `IMAGE`, and `TAG`.
+
+## Local Development
+
+### `make test`
+
+Run all unit tests with race condition checking enabled.
+
+### `make verify`
+
+Performs local verification that other than unit tests (linters, manifest updates, etc)
+
+### `make update`
+
+Updates Kustomize manifests, formatting, and tidies `go.mod`. `make verify` will ensure that `make update` was run by checking if it creates a diff.
+
+## Cluster Management
+
+### `make cluster/create`
+
+Creates a cluster for running E2E tests against. There are many parameters that can be provided via environment variables, a full list is available in [`config.sh`](../hack/e2e/config.sh), but the primary parameters are:
+
+- `CLUSTER_TYPE`: The tool used to create the cluster, either `kops` or `eksctl` - defaults to `kops`
+- `CLUSTER_NAME`: The name of the cluster to create - defaults to `ebs-csi-e2e.k8s.local`
+- `INSTANCE_TYPE`: The instance type to use for cluster nodes - defaults to `c5.large`
+- `AMI_PARAMETER`: The SSM parameter of where to get the AMI for the cluster nodes (`kops` clusters only) - defaults to `/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64`
+- `WINDOWS`: Whether or not to create a Windows node group for the cluster (`eksctl` clusters only) - defaults to `false`
+- `AWS_REGION`: Which region to create the cluster in - defaults to `us-west-2`
+- `AWS_AVAILABILITY_ZONES`: Which AZs to create nodes for the cluster in - defaults to `us-west-2a,us-west-2b,us-west-2c`
+
+#### Example: Create a default (`kops`) cluster
+
+```bash
+make cluster/create
+```
+
+#### Example: Create a cluster with only one Availability Zone
+```bash
+export AWS_AVAILABILITY_ZONES="us-west-2a"
+make cluster/create
+```
+
+#### Example: Create an `eksctl` cluster
+
+```bash
+export CLUSTER_TYPE="eksctl"
+make cluster/create
+```
+
+#### Example: Create a cluster with Graviton nodes for `arm64` testing
+
+```bash
+export INSTANCE_TYPE="m7g.medium"
+export AMI_PARAMETER="/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64"
+make cluster/create
+```
+
+#### Example: Create a cluster with Windows nodes
+
+```bash
+export WINDOWS="true"
+export CLUSTER_TYPE="eksctl"
+make cluster/create
+```
+
+### `make cluster/delete`
+
+Deletes a cluster created by `make cluster/create`. You must pass the same `CLUSTER_TYPE` and `CLUSTER_NAME` as used when creating the cluster.
+
+## E2E Tests
+
+Run E2E tests against a cluster created by `make cluster/create`. You must pass the same `CLUSTER_TYPE` and `CLUSTER_NAME` as used when creating the cluster.
+
+Alternatively, you may run on an externally created cluster by passing `CLUSTER_TYPE` (required to determine which `values.yaml` to deploy) and `KUBECONFIG`. For `kops` clusters, the node IAM role should include the appropriate IAM policies to use the driver (see [the installation docs](./install.md#set-up-driver-permissions)). For `eksctl` clusters, the `ebs-csi-controller-sa` service account should be pre-created and setup to supply an IRSA role with the appropriate policies.
+
+### `make e2e/external`
+
+Run the Kubernetes upstream [external storage E2E tests](https://github.com/kubernetes/kubernetes/blob/master/test/e2e/README.md). This is the most comprehensive E2E test, recommended for local development.
+
+### `make e2e/single-az`
+
+Run the single-AZ EBS CSI E2E tests. Requires a cluster with only one Availability Zone.
+
+### `make e2e/multi-az`
+
+Run the multi-AZ EBS CSI E2E tests. Requires a cluster with at least two Availability Zones.
+
+### `make e2e/external-arm64`
+
+Run the Kubernetes upstream [external storage E2E tests](https://github.com/kubernetes/kubernetes/blob/master/test/e2e/README.md) using an ARM64 image of the EBS CSI Driver. Requires a cluster with Graviton nodes.
+
+### `make e2e/external-windows`
+
+Run the Kubernetes upstream [external storage E2E tests](https://github.com/kubernetes/kubernetes/blob/master/test/e2e/README.md) with Windows tests enabled. Requires a cluster with Windows nodes.
+
+### `make e2e/external-kustomize`
+
+Run the Kubernetes upstream [external storage E2E tests](https://github.com/kubernetes/kubernetes/blob/master/test/e2e/README.md), but using `kustomize` to deploy the driver instead of `helm`.
+
+### `make e2e/helm-ct`
+
+Test the EBS CSI Driver Helm chart via the [Helm `chart-testing` tool](https://github.com/helm/chart-testing).
+
+## Release Scripts
+
+### `make update-sidecar-dependencies`
+
+Convenience target to perform all sidecar updates and regenerate the manifests. This is the primary target to use unless more granular control is needed.
+
+### `make update-truth-sidecars`
+
+Retrieves the latest sidecar container images and creates or updates `hack/release-scripts/image-digests.yaml`.
+
+### `make generate-sidecar-tags`
+
+Updates the Kustomize and Helm sidecar tags with the values from `hack/release-scripts/image-digests.yaml`.
diff --git a/hack/.gitignore b/hack/.gitignore
deleted file mode 100644
index 3cc5e8f05..000000000
--- a/hack/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-ebs-e2e-test/
diff --git a/hack/e2e/README.md b/hack/e2e/README.md
deleted file mode 100644
index 914f671b6..000000000
--- a/hack/e2e/README.md
+++ /dev/null
@@ -1,57 +0,0 @@
-# Usage
-
-run.sh will build and push a driver image, create a kops cluster, helm install the driver pointing to the built image, run ginkgo tests, then clean everything up.
-
-See below for an example.
-
-KOPS_STATE_FILE is an S3 bucket you have write access to.
-
-TEST_ID is a token used for idempotency.
-
-For more details, see the script itself.
-
-For more examples, see the top-level Makefile.
-
-```
-TEST_PATH=./tests/e2e-migration/... \
-EBS_CHECK_MIGRATION=true \
-TEST_ID=18512 \
-CLEAN=false \
-KOPS_STATE_FILE=s3://mattwon \
-AWS_REGION=us-west-2 \
-AWS_AVAILABILITY_ZONES=us-west-2a \
-GINKGO_FOCUS=Dynamic.\*xfs.\*should.store.data \
-GINKGO_NODES=1 \
-./hack/e2e/run.sh
-```
-
-# git read-tree
-
-Reference: https://stackoverflow.com/questions/23937436/add-subdirectory-of-remote-repo-with-git-subtree
-
-How to consume this directory by read-treeing the ebs repo:
-
-```
-git remote add ebs git@github.com:kubernetes-sigs/aws-ebs-csi-driver.git --no-tags
-git fetch ebs
-git read-tree --prefix=hack/e2e/ -u ebs/master:hack/e2e
-```
-
-To commit changes and submit them as a PR back to the ebs repo:
-
-```
-git diff ebs/master:hack/e2e HEAD:hack/e2e > /tmp/hack_e2e.diff
-pushd $GOPATH/src/github.com/kubernetes-sigs/aws-ebs-csi-driver
-git apply --reject --directory hack/e2e /tmp/hack_e2e.diff
-git commit
-```
-
-To consume newer changes from the ebs repo:
-
-```
-git fetch ebs
-git diff HEAD:hack/e2e ebs/master:hack/e2e > /tmp/hack_e2e.diff
-git apply --reject --directory hack/e2e /tmp/hack_e2e.diff
-git add hack/e2e
-git commit -m "Update hack/e2e"
-```
diff --git a/hack/e2e/chart-testing.sh b/hack/e2e/chart-testing.sh
deleted file mode 100644
index 8e7011c2e..000000000
--- a/hack/e2e/chart-testing.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -uo pipefail
-
-function ct_install() {
-  INSTALL_PATH=${1}
-  CHART_TESTING_VERSION=${2}
-  if [[ ! -e ${INSTALL_PATH}/chart-testing ]]; then
-    CHART_TESTING_DOWNLOAD_URL="https://github.com/helm/chart-testing/releases/download/v${CHART_TESTING_VERSION}/chart-testing_${CHART_TESTING_VERSION}_linux_amd64.tar.gz"
-    curl --silent --location "${CHART_TESTING_DOWNLOAD_URL}" | tar xz -C "${INSTALL_PATH}"
-    chmod +x "${INSTALL_PATH}"/ct
-  fi
-  apt-get update && apt-get install -y yamllint
-}
diff --git a/hack/e2e/config.sh b/hack/e2e/config.sh
new file mode 100644
index 000000000..b34d37db2
--- /dev/null
+++ b/hack/e2e/config.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+BASE_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
+TEST_DIR="${BASE_DIR}/csi-test-artifacts"
+mkdir -p "${TEST_DIR}"
+CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.yaml
+KUBECONFIG=${KUBECONFIG:-"${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.kubeconfig"}
+
+export AWS_REGION=${AWS_REGION:-us-west-2}
+ZONES=${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c}
+FIRST_ZONE=$(echo "${ZONES}" | cut -d, -f1)
+NODE_COUNT=${NODE_COUNT:-3}
+INSTANCE_TYPE=${INSTANCE_TYPE:-c5.large}
+WINDOWS=${WINDOWS:-"false"}
+
+# kops: must include patch version (e.g. 1.19.1)
+# eksctl: mustn't include patch version (e.g. 1.19)
+K8S_VERSION_KOPS=${K8S_VERSION_KOPS:-1.28.3}
+K8S_VERSION_EKSCTL=${K8S_VERSION_EKSCTL:-1.28}
+
+EBS_INSTALL_SNAPSHOT=${EBS_INSTALL_SNAPSHOT:-"true"}
+EBS_INSTALL_SNAPSHOT_VERSION=${EBS_INSTALL_SNAPSHOT_VERSION:-"v6.3.2"}
+
+AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+KOPS_BUCKET=${KOPS_BUCKET:-${AWS_ACCOUNT_ID}-ebs-csi-e2e-kops}
+
+AMI_PARAMETER=${AMI_PARAMETER:-/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64}
+AMI_ID=$(aws ssm get-parameters --names ${AMI_PARAMETER} --region ${AWS_REGION} --query 'Parameters[0].Value' --output text)
+
+CREATE_MISSING_ECR_REPO=${CREATE_MISSING_ECR_REPO:-"true"}
+IMAGE_NAME=${IMAGE_NAME:-${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/aws-ebs-csi-driver}
+IMAGE_TAG=${IMAGE_TAG:-$(md5sum <<<"${CLUSTER_NAME}.${CLUSTER_TYPE}" | awk '{ print $1 }')}
+IMAGE_ARCH=${IMAGE_ARCH:-amd64}
+
+DEPLOY_METHOD=${DEPLOY_METHOD:-"helm"}
+HELM_CT_TEST=${HELM_CT_TEST:-"false"}
+HELM_EXTRA_FLAGS=${HELM_EXTRA_FLAGS:-}
+COLLECT_METRICS=${COLLECT_METRICS:-"false"}
+
+TEST_PATH=${TEST_PATH:-"./tests/e2e-kubernetes/..."}
+GINKGO_FOCUS=${GINKGO_FOCUS:-"External.Storage"}
+GINKGO_SKIP=${GINKGO_SKIP:-"\[Disruptive\]|\[Serial\]"}
+GINKGO_PARALLEL=${GINKGO_PARALLEL:-25}
+
+# TODO: Left in for now, but look into if this is still necessary and remove if not
+EKSCTL_ADMIN_ROLE=${EKSCTL_ADMIN_ROLE:-"Infra-prod-KopsDeleteAllLambdaServiceRoleF1578477-1ELDFIB4KCMXV"}
diff --git a/hack/e2e/create-cluster.sh b/hack/e2e/create-cluster.sh
new file mode 100755
index 000000000..717111141
--- /dev/null
+++ b/hack/e2e/create-cluster.sh
@@ -0,0 +1,74 @@
+#!/bin/bash
+
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script creates a cluster for use of running the e2e tests
+# CLUSTER_NAME and CLUSTER_TYPE are expected to be specified by the caller
+# All other environment variables have default values (see config.sh) but
+# many can be overridden on demand if needed
+
+set -euo pipefail
+
+BASE_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
+BIN="${BASE_DIR}/../../bin"
+
+source "${BASE_DIR}/config.sh"
+source "${BASE_DIR}/util.sh"
+source "${BASE_DIR}/kops/kops.sh"
+source "${BASE_DIR}/eksctl/eksctl.sh"
+
+if [[ "${CLUSTER_TYPE}" == "kops" ]]; then
+  BUCKET_CHECK=$("${BIN}/aws" s3api head-bucket --region us-east-1 --bucket "${KOPS_BUCKET}" 2>&1 || true)
+  if grep -q "Forbidden" <<<"${BUCKET_CHECK}"; then
+    echo "Kops requires a S3 bucket in order to store the state" >&2
+    echo "This script is attempting to use a bucket called \`${KOPS_BUCKET}\`" >&2
+    echo "That bucket already exists and you do not have access to it" >&2
+    echo "You can change the bucket by setting the environment variable \$KOPS_BUCKET" >&2
+    exit 1
+  fi
+  if grep -q "Not Found" <<<"${BUCKET_CHECK}"; then
+    "${BIN}/aws" s3api create-bucket --region us-east-1 --bucket "${KOPS_BUCKET}" --acl private >/dev/null
+  fi
+
+  kops_create_cluster \
+    "$CLUSTER_NAME" \
+    "${BIN}/kops" \
+    "$ZONES" \
+    "$NODE_COUNT" \
+    "$INSTANCE_TYPE" \
+    "$AMI_ID" \
+    "$K8S_VERSION_KOPS" \
+    "$CLUSTER_FILE" \
+    "$KUBECONFIG" \
+    "${BASE_DIR}/kops/patch-cluster.yaml" \
+    "${BASE_DIR}/kops/patch-node.yaml" \
+    "s3://${KOPS_BUCKET}"
+elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
+  eksctl_create_cluster \
+    "$CLUSTER_NAME" \
+    "${BIN}/eksctl" \
+    "$ZONES" \
+    "$INSTANCE_TYPE" \
+    "$K8S_VERSION_EKSCTL" \
+    "$CLUSTER_FILE" \
+    "$KUBECONFIG" \
+    "${BASE_DIR}/eksctl/patch.yaml" \
+    "$EKSCTL_ADMIN_ROLE" \
+    "$WINDOWS" \
+    "${BASE_DIR}/eksctl/vpc-resource-controller-configmap.yaml"
+else
+  echo "Cluster type ${CLUSTER_TYPE} is invalid, must be kops or eksctl" >&2
+  exit 1
+fi
diff --git a/hack/e2e/delete-cluster.sh b/hack/e2e/delete-cluster.sh
new file mode 100755
index 000000000..dca150d0c
--- /dev/null
+++ b/hack/e2e/delete-cluster.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script deletes a cluster that was created by `create-cluster.sh`
+# CLUSTER_NAME and CLUSTER_TYPE are expected to be specified by the caller
+# All other environment variables have default values (see config.sh) but
+# many can be overridden on demand if needed
+
+set -euo pipefail
+
+BASE_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
+BIN="${BASE_DIR}/../../bin"
+
+source "${BASE_DIR}/config.sh"
+source "${BASE_DIR}/util.sh"
+source "${BASE_DIR}/kops/kops.sh"
+source "${BASE_DIR}/eksctl/eksctl.sh"
+
+if [[ "${CLUSTER_TYPE}" == "kops" ]]; then
+  kops_delete_cluster \
+    "${BIN}/kops" \
+    "${CLUSTER_NAME}" \
+    "s3://${KOPS_BUCKET}"
+elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
+  eksctl_delete_cluster \
+    "${BIN}/eksctl" \
+    "${CLUSTER_NAME}"
+else
+  echo "Cluster type ${CLUSTER_TYPE} is invalid, must be kops or eksctl" >&2
+  exit 1
+fi
diff --git a/hack/e2e/ecr.sh b/hack/e2e/ecr.sh
index dc82f1ae2..9f5e4f0d2 100644
--- a/hack/e2e/ecr.sh
+++ b/hack/e2e/ecr.sh
@@ -1,9 +1,20 @@
 #!/bin/bash
 
-set -uo pipefail
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
-BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
-source "${BASE_DIR}"/util.sh
+set -euo pipefail
 
 function ecr_build_and_push() {
   REGION=${1}
@@ -11,26 +22,27 @@ function ecr_build_and_push() {
   IMAGE_NAME=${3}
   IMAGE_TAG=${4}
   IMAGE_ARCH=${5}
-  set +e
-  if docker images --format "{{.Repository}}:{{.Tag}}" | grep "${IMAGE_NAME}:${IMAGE_TAG}"; then
-    set -e
-    loudecho "Assuming ${IMAGE_NAME}:${IMAGE_TAG} has been built and pushed"
+
+  loudecho "Building and pushing test driver image to ${IMAGE_NAME}:${IMAGE_TAG}"
+  aws ecr get-login-password --region "${REGION}" | docker login --username AWS --password-stdin "${AWS_ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com"
+
+  # Only setup buildx builder on Prow, allow local users to use docker cache
+  if [ -n "${PROW_JOB_ID:-}" ]; then
+    trap "docker buildx rm ebs-csi-multiarch-builder" EXIT
+    docker buildx create --bootstrap --use --name ebs-csi-multiarch-builder
+    docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+  fi
+
+  export IMAGE="${IMAGE_NAME}"
+  export TAG="${IMAGE_TAG}"
+  if [[ "$WINDOWS" == true ]]; then
+    export ALL_OS="linux windows"
+    export ALL_OSVERSION_windows="ltsc2022"
+    export ALL_ARCH_linux="amd64"
+    export ALL_ARCH_windows="${IMAGE_ARCH}"
   else
-    set -e
-    loudecho "Building and pushing test driver image to ${IMAGE_NAME}:${IMAGE_TAG}"
-    aws ecr get-login-password --region "${REGION}" | docker login --username AWS --password-stdin "${AWS_ACCOUNT_ID}".dkr.ecr."${REGION}".amazonaws.com
-    if [[ "$WINDOWS" == true ]]; then
-      export DOCKER_CLI_EXPERIMENTAL=enabled
-      export TAG=${IMAGE_TAG}
-      export IMAGE=${IMAGE_NAME}
-      trap "docker buildx rm multiarch-builder" EXIT
-      docker buildx create --use --name multiarch-builder
-      docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-      make all-push
-    else
-      IMAGE=${IMAGE_NAME} TAG=${IMAGE_TAG} OS=linux ARCH=${IMAGE_ARCH} OSVERSION=al2023 make image
-      docker tag "${IMAGE_NAME}":"${IMAGE_TAG}"-linux-${IMAGE_ARCH}-al2023 "${IMAGE_NAME}":"${IMAGE_TAG}"
-      docker push "${IMAGE_NAME}":"${IMAGE_TAG}"
-    fi
+    export ALL_OS="linux"
+    export ALL_ARCH_linux="${IMAGE_ARCH}"
   fi
+  make -j $(nproc) all-push
 }
diff --git a/hack/e2e/eksctl.sh b/hack/e2e/eksctl/eksctl.sh
similarity index 59%
rename from hack/e2e/eksctl.sh
rename to hack/e2e/eksctl/eksctl.sh
index 6d448dc84..4b2ccaf68 100644
--- a/hack/e2e/eksctl.sh
+++ b/hack/e2e/eksctl/eksctl.sh
@@ -1,20 +1,27 @@
 #!/bin/bash
 
-set -euo pipefail
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script updates the kustomize templates in deploy/kubernetes/base/ by
+# running `helm template` and stripping the namespace from the output
 
-function eksctl_install() {
-  INSTALL_PATH=${1}
-  EKSCTL_VERSION=${2}
-  if [[ ! -e ${INSTALL_PATH}/eksctl ]]; then
-    EKSCTL_DOWNLOAD_URL="https://github.com/weaveworks/eksctl/releases/download/v${EKSCTL_VERSION}/eksctl_$(uname -s)_amd64.tar.gz"
-    curl --silent --location "${EKSCTL_DOWNLOAD_URL}" | tar xz -C "${INSTALL_PATH}"
-    chmod +x "${INSTALL_PATH}"/eksctl
-  fi
-}
+set -euo pipefail
 
 function eksctl_create_cluster() {
   CLUSTER_NAME=${1}
-  BIN=${2}
+  EKSCTL_BIN=${2}
   ZONES=${3}
   INSTANCE_TYPE=${4}
   K8S_VERSION=${5}
@@ -27,12 +34,12 @@ function eksctl_create_cluster() {
 
   CLUSTER_NAME="${CLUSTER_NAME//./-}"
 
-  if eksctl_cluster_exists "${CLUSTER_NAME}" "${BIN}"; then
+  if eksctl_cluster_exists "${CLUSTER_NAME}" "${EKSCTL_BIN}"; then
     loudecho "Upgrading cluster $CLUSTER_NAME with $CLUSTER_FILE"
-    ${BIN} upgrade cluster -f "${CLUSTER_FILE}"
+    ${EKSCTL_BIN} upgrade cluster -f "${CLUSTER_FILE}"
   else
     loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE (dry run)"
-    ${BIN} create cluster \
+    ${EKSCTL_BIN} create cluster \
       --managed \
       --ssh-access=false \
       --zones "${ZONES}" \
@@ -41,29 +48,29 @@ function eksctl_create_cluster() {
       --version="${K8S_VERSION}" \
       --disable-pod-imds \
       --dry-run \
-      "${CLUSTER_NAME}" > "${CLUSTER_FILE}"
+      "${CLUSTER_NAME}" >"${CLUSTER_FILE}"
 
     if test -f "$EKSCTL_PATCH_FILE"; then
       eksctl_patch_cluster_file "$CLUSTER_FILE" "$EKSCTL_PATCH_FILE"
     fi
 
     loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE"
-    ${BIN} create cluster -f "${CLUSTER_FILE}" --kubeconfig "${KUBECONFIG}"
+    ${EKSCTL_BIN} create cluster -f "${CLUSTER_FILE}" --kubeconfig "${KUBECONFIG}"
   fi
 
   loudecho "Cluster ${CLUSTER_NAME} kubecfg written to ${KUBECONFIG}"
   loudecho "Getting cluster ${CLUSTER_NAME}"
-  ${BIN} get cluster "${CLUSTER_NAME}"
+  ${EKSCTL_BIN} get cluster "${CLUSTER_NAME}"
 
   if [[ -n "$EKSCTL_ADMIN_ROLE" ]]; then
     AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
     ADMIN_ARN="arn:aws:iam::${AWS_ACCOUNT_ID}:role/${EKSCTL_ADMIN_ROLE}"
     loudecho "Granting ${ADMIN_ARN} admin access to the cluster"
-    ${BIN} create iamidentitymapping --cluster "${CLUSTER_NAME}" --arn "${ADMIN_ARN}" --group system:masters --username admin
+    ${EKSCTL_BIN} create iamidentitymapping --cluster "${CLUSTER_NAME}" --arn "${ADMIN_ARN}" --group system:masters --username admin
   fi
 
   if [[ "$WINDOWS" == true ]]; then
-    ${BIN} create nodegroup \
+    ${EKSCTL_BIN} create nodegroup \
       --managed=true \
       --ssh-access=false \
       --cluster="${CLUSTER_NAME}" \
@@ -71,7 +78,7 @@ function eksctl_create_cluster() {
       --instance-types=m5.2xlarge \
       -n ng-windows \
       -m 3 \
-      -M 3 \
+      -M 3
 
     kubectl apply --kubeconfig "${KUBECONFIG}" -f "$VPC_CONFIGMAP_FILE"
   fi
@@ -81,9 +88,9 @@ function eksctl_create_cluster() {
 
 function eksctl_cluster_exists() {
   CLUSTER_NAME=${1}
-  BIN=${2}
+  EKSCTL_BIN=${2}
   set +e
-  if ${BIN} get cluster "${CLUSTER_NAME}"; then
+  if ${EKSCTL_BIN} get cluster "${CLUSTER_NAME}"; then
     set -e
     return 0
   else
@@ -93,10 +100,13 @@ function eksctl_cluster_exists() {
 }
 
 function eksctl_delete_cluster() {
-  BIN=${1}
+  EKSCTL_BIN=${1}
   CLUSTER_NAME=${2}
+
+  CLUSTER_NAME="${CLUSTER_NAME//./-}"
+
   loudecho "Deleting cluster ${CLUSTER_NAME}"
-  ${BIN} delete cluster "${CLUSTER_NAME}"
+  ${EKSCTL_BIN} delete cluster "${CLUSTER_NAME}"
 }
 
 function eksctl_patch_cluster_file() {
@@ -112,7 +122,7 @@ function eksctl_patch_cluster_file() {
   cp "$CLUSTER_FILE" "$CLUSTER_FILE_0"
 
   # Patch only the Cluster
-  kubectl patch -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$EKSCTL_PATCH_FILE")" -o yaml > "$CLUSTER_FILE_1"
+  kubectl patch --kubeconfig "/dev/null" -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$EKSCTL_PATCH_FILE")" -o yaml >"$CLUSTER_FILE_1"
   mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
 
   # Done patching, overwrite original CLUSTER_FILE
diff --git a/hack/eksctl-patch.yaml b/hack/e2e/eksctl/patch.yaml
similarity index 100%
rename from hack/eksctl-patch.yaml
rename to hack/e2e/eksctl/patch.yaml
diff --git a/hack/values_eksctl.yaml b/hack/e2e/eksctl/values.yaml
similarity index 100%
rename from hack/values_eksctl.yaml
rename to hack/e2e/eksctl/values.yaml
diff --git a/hack/vpc-resource-controller-configmap.yaml b/hack/e2e/eksctl/vpc-resource-controller-configmap.yaml
similarity index 100%
rename from hack/vpc-resource-controller-configmap.yaml
rename to hack/e2e/eksctl/vpc-resource-controller-configmap.yaml
diff --git a/hack/e2e/helm.sh b/hack/e2e/helm.sh
deleted file mode 100644
index 500441e1a..000000000
--- a/hack/e2e/helm.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-
-set -uo pipefail
-
-function helm_install() {
-  INSTALL_PATH=${1}
-  if [[ ! -e ${INSTALL_PATH}/helm ]]; then
-    curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
-    chmod 700 get_helm.sh
-    export USE_SUDO=false
-    export HELM_INSTALL_DIR=${INSTALL_PATH}
-    ./get_helm.sh
-    rm get_helm.sh
-  fi
-}
diff --git a/hack/e2e/kops.sh b/hack/e2e/kops/kops.sh
similarity index 64%
rename from hack/e2e/kops.sh
rename to hack/e2e/kops/kops.sh
index b5e5dade4..4f34c3580 100644
--- a/hack/e2e/kops.sh
+++ b/hack/e2e/kops/kops.sh
@@ -1,30 +1,27 @@
 #!/bin/bash
 
-set -euo pipefail
-
-OS_ARCH=$(go env GOOS)-amd64
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script updates the kustomize templates in deploy/kubernetes/base/ by
+# running `helm template` and stripping the namespace from the output
 
-BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
-source "${BASE_DIR}"/util.sh
-
-function kops_install() {
-  INSTALL_PATH=${1}
-  KOPS_VERSION=${2}
-  if [[ -e "${INSTALL_PATH}"/kops ]]; then
-    INSTALLED_KOPS_VERSION=$("${INSTALL_PATH}"/kops version)
-    if [[ "$INSTALLED_KOPS_VERSION" == *"$KOPS_VERSION"* ]]; then
-      echo "KOPS $INSTALLED_KOPS_VERSION already installed!"
-      return
-    fi
-  fi
-  KOPS_DOWNLOAD_URL=https://github.com/kubernetes/kops/releases/download/v${KOPS_VERSION}/kops-${OS_ARCH}
-  curl -L -X GET "${KOPS_DOWNLOAD_URL}" -o "${INSTALL_PATH}"/kops
-  chmod +x "${INSTALL_PATH}"/kops
-}
+set -euo pipefail
 
 function kops_create_cluster() {
   CLUSTER_NAME=${1}
-  BIN=${2}
+  KOPS_BIN=${2}
   ZONES=${3}
   NODE_COUNT=${4}
   INSTANCE_TYPE=${5}
@@ -36,12 +33,12 @@ function kops_create_cluster() {
   KOPS_PATCH_NODE_FILE=${11}
   KOPS_STATE_FILE=${12}
 
-  if kops_cluster_exists "${CLUSTER_NAME}" "${BIN}" "${KOPS_STATE_FILE}"; then
+  if kops_cluster_exists "${CLUSTER_NAME}" "${KOPS_BIN}" "${KOPS_STATE_FILE}"; then
     loudecho "Replacing cluster $CLUSTER_NAME with $CLUSTER_FILE"
-    ${BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
+    ${KOPS_BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
   else
     loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE (dry run)"
-    ${BIN} create cluster --state "${KOPS_STATE_FILE}" \
+    ${KOPS_BIN} create cluster --state "${KOPS_STATE_FILE}" \
       --zones "${ZONES}" \
       --node-count="${NODE_COUNT}" \
       --node-size="${INSTANCE_TYPE}" \
@@ -49,7 +46,7 @@ function kops_create_cluster() {
       --kubernetes-version="${K8S_VERSION}" \
       --dry-run \
       -o yaml \
-      "${CLUSTER_NAME}" > "${CLUSTER_FILE}"
+      "${CLUSTER_NAME}" >"${CLUSTER_FILE}"
 
     if test -f "$KOPS_PATCH_FILE"; then
       kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_FILE" "Cluster" ""
@@ -59,26 +56,26 @@ function kops_create_cluster() {
     fi
 
     loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE"
-    ${BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
+    ${KOPS_BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
   fi
 
   loudecho "Updating cluster $CLUSTER_NAME with $CLUSTER_FILE"
-  ${BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes
+  ${KOPS_BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes
 
   loudecho "Exporting cluster ${CLUSTER_NAME} kubecfg to ${KUBECONFIG}"
-  ${BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}"
+  ${KOPS_BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}"
 
   loudecho "Validating cluster ${CLUSTER_NAME}"
-  ${BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m --kubeconfig "${KUBECONFIG}"
+  ${KOPS_BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m --kubeconfig "${KUBECONFIG}"
   return $?
 }
 
 function kops_cluster_exists() {
   CLUSTER_NAME=${1}
-  BIN=${2}
+  KOPS_BIN=${2}
   KOPS_STATE_FILE=${3}
   set +e
-  if ${BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then
+  if ${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then
     set -e
     return 0
   else
@@ -88,11 +85,11 @@ function kops_cluster_exists() {
 }
 
 function kops_delete_cluster() {
-  BIN=${1}
+  KOPS_BIN=${1}
   CLUSTER_NAME=${2}
   KOPS_STATE_FILE=${3}
   loudecho "Deleting cluster ${CLUSTER_NAME}"
-  ${BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes
+  ${KOPS_BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes
 }
 
 # TODO switch this to python, work exclusively with yaml, use kops toolbox
@@ -119,15 +116,15 @@ function kops_patch_cluster_file() {
   if [ -n "$ROLE" ]; then
     FILTER="$FILTER | select(.spec.role==\"$ROLE\")"
   fi
-  jq "$FILTER" "$CLUSTER_FILE_JSON" > "$CLUSTER_FILE_0"
+  jq "$FILTER" "$CLUSTER_FILE_JSON" >"$CLUSTER_FILE_0"
 
   # Patch only the json objects
-  kubectl patch -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$KOPS_PATCH_FILE")" -o json > "$CLUSTER_FILE_1"
+  kubectl patch -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$KOPS_PATCH_FILE")" -o json >"$CLUSTER_FILE_1"
   mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
 
   # Delete the original json objects, add the patched
   # TODO Cluster must always be first?
-  jq "del($FILTER)" "$CLUSTER_FILE_JSON" | jq ". + \$patched | sort" --slurpfile patched "$CLUSTER_FILE_0" > "$CLUSTER_FILE_1"
+  jq "del($FILTER)" "$CLUSTER_FILE_JSON" | jq ". + \$patched | sort" --slurpfile patched "$CLUSTER_FILE_0" >"$CLUSTER_FILE_1"
   mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
 
   # HACK convert the array of json objects to multiple yaml documents
@@ -144,14 +141,14 @@ function kops_patch_cluster_file() {
 function yaml_to_json() {
   IN=${1}
   OUT=${2}
-  kubectl patch -f "$IN" --local -p "{}" --type merge -o json | jq '.' -s > "$OUT"
+  kubectl patch -f "$IN" --local -p "{}" --type merge -o json | jq '.' -s >"$OUT"
 }
 
 function json_to_yaml() {
   IN=${1}
   OUT=${2}
   for ((i = 0; i < $(jq length "$IN"); i++)); do
-    echo "---" >> "$OUT"
-    jq ".[$i]" "$IN" | kubectl patch -f - --local -p "{}" --type merge -o yaml >> "$OUT"
+    echo "---" >>"$OUT"
+    jq ".[$i]" "$IN" | kubectl patch -f - --local -p "{}" --type merge -o yaml >>"$OUT"
   done
 }
diff --git a/hack/kops-patch.yaml b/hack/e2e/kops/patch-cluster.yaml
similarity index 100%
rename from hack/kops-patch.yaml
rename to hack/e2e/kops/patch-cluster.yaml
diff --git a/hack/kops-patch-node.yaml b/hack/e2e/kops/patch-node.yaml
similarity index 100%
rename from hack/kops-patch-node.yaml
rename to hack/e2e/kops/patch-node.yaml
diff --git a/hack/values.yaml b/hack/e2e/kops/values.yaml
similarity index 100%
rename from hack/values.yaml
rename to hack/e2e/kops/values.yaml
diff --git a/hack/e2e/metrics/metrics.sh b/hack/e2e/metrics/metrics.sh
index a01ac8a28..a0faab4d4 100644
--- a/hack/e2e/metrics/metrics.sh
+++ b/hack/e2e/metrics/metrics.sh
@@ -46,9 +46,9 @@ check_dependencies() {
 collect_metrics() {
   log "Collecting metrics in $METRICS_DIR_PATH"
   mkdir -p "$METRICS_DIR_PATH"
-  
+
   log "Collecting deployment time"
-  echo -e "$DEPLOYMENT_TIME" > "$METRICS_DIR_PATH/deployment_time.txt"
+  echo -e "$DEPLOYMENT_TIME" >"$METRICS_DIR_PATH/deployment_time.txt"
 
   log "Collecting resource metrics"
   install_metrics_server
@@ -95,7 +95,7 @@ collect_resource_metrics() {
   local readonly label="$2"
   local readonly output_file="$3"
 
-  kubectl get PodMetrics --kubeconfig "$KUBECONFIG" -n "${namespace}" -l "${label}" -o yaml > "${output_file}"
+  kubectl get PodMetrics --kubeconfig "$KUBECONFIG" -n "${namespace}" -l "${label}" -o yaml >"${output_file}"
 }
 
 collect_clusterloader2_metrics() {
@@ -112,7 +112,7 @@ collect_clusterloader2_metrics() {
     --repo-root="$PERF_TESTS_DIR" \
     --test-configs="$CLUSTER_LOADER_CONFIG" \
     --test-overrides="$CLUSTER_LOADER_OVERRIDE" \
-    --report-dir="$METRICS_DIR_PATH" \
+    --report-dir="$METRICS_DIR_PATH"
 
   local readonly exit_code=$?
   if [[ ${exit_code} -ne 0 ]]; then
@@ -161,9 +161,9 @@ upload_metrics() {
   }'
 
   aws s3api put-bucket-policy \
-  --bucket "$SOURCE_BUCKET" \
-  --policy "$bucket_policy" \
-  --region "$AWS_REGION"
+    --bucket "$SOURCE_BUCKET" \
+    --policy "$bucket_policy" \
+    --region "$AWS_REGION"
 
   log "Setting lifecycle policy on S3 bucket $SOURCE_BUCKET"
   local readonly lifecycle_policy='{
diff --git a/hack/e2e/run.sh b/hack/e2e/run.sh
index a083394d1..1866d355f 100755
--- a/hack/e2e/run.sh
+++ b/hack/e2e/run.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2019 The Kubernetes Authors.
+# Copyright 2023 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,168 +14,56 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -euo pipefail
-
-BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
-source "${BASE_DIR}"/ecr.sh
-source "${BASE_DIR}"/eksctl.sh
-source "${BASE_DIR}"/helm.sh
-source "${BASE_DIR}"/kops.sh
-source "${BASE_DIR}"/util.sh
-source "${BASE_DIR}"/chart-testing.sh
-source "${BASE_DIR}"/metrics/metrics.sh
-
-DRIVER_NAME=${DRIVER_NAME:-aws-ebs-csi-driver}
-CONTAINER_NAME=${CONTAINER_NAME:-ebs-plugin}
-
-TEST_ID=${TEST_ID:-$RANDOM}
-CLUSTER_NAME=test-cluster-${TEST_ID}.k8s.local
-CLUSTER_TYPE=${CLUSTER_TYPE:-kops}
-
-TEST_DIR=${BASE_DIR}/csi-test-artifacts
-BIN_DIR=${TEST_DIR}/bin
-CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.yaml
-KUBECONFIG=${KUBECONFIG:-"${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.kubeconfig"}
-
-REGION=${AWS_REGION:-us-west-2}
-ZONES=${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c}
-FIRST_ZONE=$(echo "${ZONES}" | cut -d, -f1)
-NODE_COUNT=${NODE_COUNT:-3}
-INSTANCE_TYPE=${INSTANCE_TYPE:-c5.large}
-
-AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
-IMAGE_NAME=${IMAGE_NAME:-${AWS_ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/${DRIVER_NAME}}
-IMAGE_TAG=${IMAGE_TAG:-${TEST_ID}}
-IMAGE_ARCH=${IMAGE_ARCH:-amd64}
-
-# kops: must include patch version (e.g. 1.19.1)
-# eksctl: mustn't include patch version (e.g. 1.19)
-K8S_VERSION_KOPS=${K8S_VERSION_KOPS:-${K8S_VERSION:-1.28.3}}
-K8S_VERSION_EKSCTL=${K8S_VERSION_EKSCTL:-${K8S_VERSION:-1.28}}
-
-KOPS_VERSION=${KOPS_VERSION:-1.28.0}
-KOPS_STATE_FILE=${KOPS_STATE_FILE:-s3://k8s-kops-csi-shared-e2e}
-KOPS_PATCH_FILE=${KOPS_PATCH_FILE:-./hack/kops-patch.yaml}
-KOPS_PATCH_NODE_FILE=${KOPS_PATCH_NODE_FILE:-./hack/kops-patch-node.yaml}
-AMI_PARAMETER=${AMI_PARAMETER:-/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64}
-AMI_ID=$(aws ssm get-parameters --names ${AMI_PARAMETER} --region ${REGION} --query 'Parameters[0].Value' --output text)
-
-EKSCTL_VERSION=${EKSCTL_VERSION:-0.164.0}
-EKSCTL_PATCH_FILE=${EKSCTL_PATCH_FILE:-./hack/eksctl-patch.yaml}
-VPC_CONFIGMAP_FILE=${VPC_CONFIGMAP_FILE:-./hack/vpc-resource-controller-configmap.yaml}
-EKSCTL_ADMIN_ROLE=${EKSCTL_ADMIN_ROLE:-}
-# Creates a windows node group.
-WINDOWS=${WINDOWS:-"false"}
+# This script builds and deploys the EBS CSI Driver and runs e2e tests
+# CLUSTER_NAME and CLUSTER_TYPE are expected to be specified by the caller
+# All other environment variables have default values (see config.sh) but
+# many can be overridden on demand if needed
 
-# Valid deploy methods: "helm" (default), "kustomize"
-DEPLOY_METHOD=${DEPLOY_METHOD:-"helm"}
-
-HELM_VALUES_FILE=${HELM_VALUES_FILE:-./hack/values.yaml}
-HELM_EXTRA_FLAGS=${HELM_EXTRA_FLAGS:-}
-
-TEST_PATH=${TEST_PATH:-"./tests/e2e/..."}
-ARTIFACTS=${ARTIFACTS:-"${TEST_DIR}/artifacts"}
-GINKGO_FOCUS=${GINKGO_FOCUS:-"\[ebs-csi-e2e\]"}
-GINKGO_SKIP=${GINKGO_SKIP:-"\[Disruptive\]"}
-GINKGO_NODES=${GINKGO_NODES:-4}
-GINKGO_PARALLEL=${GINKGO_PARALLEL:-25}
-NODE_OS_DISTRO=${NODE_OS_DISTRO:-"linux"}
-TEST_EXTRA_FLAGS=${TEST_EXTRA_FLAGS:-}
-
-EBS_INSTALL_SNAPSHOT=${EBS_INSTALL_SNAPSHOT:-"false"}
-# https://github.com/kubernetes-csi/external-snapshotter
-EBS_INSTALL_SNAPSHOT_VERSION=${EBS_INSTALL_SNAPSHOT_VERSION:-"v6.3.2"}
+set -euo pipefail
 
-HELM_CT_TEST=${HELM_CT_TEST:-"false"}
-# https://github.com/helm/chart-testing
-CHART_TESTING_VERSION=${CHART_TESTING_VERSION:-3.8.0}
-CLEAN=${CLEAN:-"true"}
+BASE_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
+BIN="${BASE_DIR}/../../bin"
 
-COLLECT_METRICS=${COLLECT_METRICS:-"false"}
+source "${BASE_DIR}/config.sh"
+source "${BASE_DIR}/util.sh"
+source "${BASE_DIR}/ecr.sh"
+source "${BASE_DIR}/metrics/metrics.sh"
 
-loudecho "Testing in region ${REGION} and zones ${ZONES}"
-mkdir -p "${BIN_DIR}"
-export PATH=${PATH}:${BIN_DIR}
+## Setup
 
 if [[ "${CLUSTER_TYPE}" == "kops" ]]; then
-  loudecho "Installing kops ${KOPS_VERSION} to ${BIN_DIR}"
-  kops_install "${BIN_DIR}" "${KOPS_VERSION}"
-  KOPS_BIN=${BIN_DIR}/kops
+  HELM_VALUES_FILE="${BASE_DIR}/kops/values.yaml"
+  K8S_VERSION="${K8S_VERSION_KOPS}"
 elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
-  loudecho "Installing eksctl ${EKSCTL_VERSION} to ${BIN_DIR}"
-  eksctl_install "${BIN_DIR}" "${EKSCTL_VERSION}"
-  EKSCTL_BIN=${BIN_DIR}/eksctl
+  HELM_VALUES_FILE="${BASE_DIR}/eksctl/values.yaml"
+  K8S_VERSION="${K8S_VERSION_EKSCTL}"
 else
-  loudecho "${CLUSTER_TYPE} must be kops or eksctl!"
+  echo "Cluster type ${CLUSTER_TYPE} is invalid, must be kops or eksctl" >&2
   exit 1
 fi
 
-loudecho "Installing helm to ${BIN_DIR}"
-helm_install "${BIN_DIR}"
-HELM_BIN=${BIN_DIR}/helm
-
-if [[ "${HELM_CT_TEST}" == true ]]; then
-  loudecho "Installing chart-testing ${CHART_TESTING_VERSION} to ${BIN_DIR}"
-  ct_install "${BIN_DIR}" "${CHART_TESTING_VERSION}"
-  CHART_TESTING_BIN=${BIN_DIR}/ct
+if [[ "$WINDOWS" == true ]]; then
+  NODE_OS_DISTRO="windows"
 else
-  loudecho "Installing ginkgo to ${BIN_DIR}"
-  GINKGO_BIN=${BIN_DIR}/ginkgo
-  if [[ ! -e ${GINKGO_BIN} ]]; then
-    pushd /tmp
-    GOPATH=${TEST_DIR} GOBIN=${BIN_DIR} go install github.com/onsi/ginkgo/v2/ginkgo@v2.11.0
-    popd
-    ginkgo version
-  fi
-  loudecho "Installing kubetest2 to ${BIN_DIR}"
-  KUBETEST2_BIN=${BIN_DIR}/kubetest2
-  if [[ ! -e ${KUBETEST2_BIN} ]]; then
-    pushd /tmp
-    GOPATH=${TEST_DIR} GOBIN=${BIN_DIR} go install sigs.k8s.io/kubetest2/...@latest
-    popd
+  NODE_OS_DISTRO="linux"
+fi
+
+## Build image
+
+if [[ "${CREATE_MISSING_ECR_REPO}" == true ]]; then
+  REPO_CHECK=$(aws ecr describe-repositories --region "${AWS_REGION}")
+  if [ $(jq '.repositories | map(.repositoryName) | index("aws-ebs-csi-driver")' <<<"${REPO_CHECK}") == "null" ]; then
+    aws ecr create-repository --region "${AWS_REGION}" --repository-name aws-ebs-csi-driver >/dev/null
   fi
 fi
 
-ecr_build_and_push "${REGION}" \
+ecr_build_and_push "${AWS_REGION}" \
   "${AWS_ACCOUNT_ID}" \
   "${IMAGE_NAME}" \
   "${IMAGE_TAG}" \
   "${IMAGE_ARCH}"
 
-if [[ "${CLUSTER_TYPE}" == "kops" ]]; then
-  kops_create_cluster \
-    "$CLUSTER_NAME" \
-    "$KOPS_BIN" \
-    "$ZONES" \
-    "$NODE_COUNT" \
-    "$INSTANCE_TYPE" \
-    "$AMI_ID" \
-    "$K8S_VERSION_KOPS" \
-    "$CLUSTER_FILE" \
-    "$KUBECONFIG" \
-    "$KOPS_PATCH_FILE" \
-    "$KOPS_PATCH_NODE_FILE" \
-    "$KOPS_STATE_FILE"
-  if [[ $? -ne 0 ]]; then
-    exit 1
-  fi
-elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
-  eksctl_create_cluster \
-    "$CLUSTER_NAME" \
-    "$EKSCTL_BIN" \
-    "$ZONES" \
-    "$INSTANCE_TYPE" \
-    "$K8S_VERSION_EKSCTL" \
-    "$CLUSTER_FILE" \
-    "$KUBECONFIG" \
-    "$EKSCTL_PATCH_FILE" \
-    "$EKSCTL_ADMIN_ROLE" \
-    "$WINDOWS" \
-    "$VPC_CONFIGMAP_FILE"
-  if [[ $? -ne 0 ]]; then
-    exit 1
-  fi
-fi
+## Deploy
 
 if [[ "${EBS_INSTALL_SNAPSHOT}" == true ]]; then
   loudecho "Installing snapshot controller and CRDs"
@@ -186,37 +74,20 @@ if [[ "${EBS_INSTALL_SNAPSHOT}" == true ]]; then
   kubectl apply --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml
 fi
 
-if [[ "${HELM_CT_TEST}" == true ]]; then
-  loudecho "Test and lint Helm chart with chart-testing"
-  if [ -n "${PROW_JOB_ID:-}" ]; then
-    # Prow-specific setup
-    # Required becuase chart_testing ALWAYS needs a remote
-    git remote add ct https://github.com/kubernetes-sigs/aws-ebs-csi-driver.git
-    git fetch ct "${PULL_BASE_REF}"
-    export CT_REMOTE="ct"
-    export CT_TARGET_BRANCH="${PULL_BASE_REF}"
-  fi
-  set -x
-  set +e
-  export KUBECONFIG="${KUBECONFIG}"
-  ${CHART_TESTING_BIN} lint-and-install --config ${PWD}/tests/ct-config.yaml --helm-extra-set-args="--set=image.repository=${IMAGE_NAME},image.tag=${IMAGE_TAG},node.tolerateAllTaints=false"
-  TEST_PASSED=$?
-  set -e
-  set +x
-else
-  loudecho "Deploying driver via ${DEPLOY_METHOD}"
+if [[ "${HELM_CT_TEST}" != true ]]; then
   startSec=$(date +'%s')
 
   if [[ ${DEPLOY_METHOD} == "helm" ]]; then
-    HELM_ARGS=(upgrade --install "${DRIVER_NAME}"
+    HELM_ARGS=(upgrade --install "aws-ebs-csi-driver"
       --namespace kube-system
       --set image.repository="${IMAGE_NAME}"
       --set image.tag="${IMAGE_TAG}"
       --set node.enableWindows="${WINDOWS}"
+      --set=controller.k8sTagClusterId="${CLUSTER_NAME}"
       --timeout 10m0s
       --wait
       --kubeconfig "${KUBECONFIG}"
-      ./charts/"${DRIVER_NAME}")
+      ./charts/aws-ebs-csi-driver)
     if [[ -f "$HELM_VALUES_FILE" ]]; then
       HELM_ARGS+=(-f "${HELM_VALUES_FILE}")
     fi
@@ -225,34 +96,58 @@ else
       HELM_ARGS+=("${EXPANDED_HELM_EXTRA_FLAGS}")
     fi
     set -x
-    "${HELM_BIN}" "${HELM_ARGS[@]}"
+    "${BIN}/helm" "${HELM_ARGS[@]}"
     set +x
   elif [[ ${DEPLOY_METHOD} == "kustomize" ]]; then
+    set -x
     kubectl --kubeconfig "${KUBECONFIG}" apply -k "./deploy/kubernetes/overlays/stable"
     kubectl --kubeconfig "${KUBECONFIG}" --namespace kube-system wait --timeout 10m0s --for "condition=ready" pod -l "app.kubernetes.io/name=aws-ebs-csi-driver"
+    set +x
   fi
 
   endSec=$(date +'%s')
   deployTimeSeconds=$(((endSec - startSec) / 1))
   loudecho "Driver deployment complete, time used: $deployTimeSeconds seconds"
+fi
+
+## Run tests
+
+if [[ "${HELM_CT_TEST}" == true ]]; then
+  loudecho "Test and lint Helm chart with chart-testing"
+  if [ -n "${PROW_JOB_ID:-}" ]; then
+    # Prow-specific setup
+    # Required becuase chart_testing ALWAYS needs a remote
+    git remote add ct https://github.com/kubernetes-sigs/aws-ebs-csi-driver.git
+    git fetch ct "${PULL_BASE_REF}"
+    export CT_REMOTE="ct"
+    export CT_TARGET_BRANCH="${PULL_BASE_REF}"
+  fi
+  set -x
+  set +e
+  KUBECONFIG="$KUBECONFIG" PATH="${BIN}:${PATH}" "${BIN}/ct" lint-and-install \
+    --config="${BASE_DIR}/../../tests/ct-config.yaml" \
+    --helm-extra-set-args="--set=image.repository=${IMAGE_NAME},image.tag=${IMAGE_TAG},node.tolerateAllTaints=false"
+  TEST_PASSED=$?
+  set -e
+  set +x
+else
   loudecho "Testing focus ${GINKGO_FOCUS}"
 
   if [[ $TEST_PATH == "./tests/e2e-kubernetes/..." ]]; then
-    pushd ${PWD}/tests/e2e-kubernetes
-    packageVersion=$(echo $(cut -d '.' -f 1,2 <<< $K8S_VERSION))
+    pushd "${BASE_DIR}/../../tests/e2e-kubernetes"
+    packageVersion=$(echo $(cut -d '.' -f 1,2 <<<$K8S_VERSION))
 
     set -x
     set +e
-    kubetest2 noop \
+    "${BIN}/kubetest2" noop \
       --run-id="e2e-kubernetes" \
       --test=ginkgo \
       -- \
       --skip-regex="${GINKGO_SKIP}" \
       --focus-regex="${GINKGO_FOCUS}" \
-      --test-package-version=$(curl -L https://dl.k8s.io/release/stable-$packageVersion.txt) \
+      --test-package-version=$(curl -L https://dl.k8s.io/release/stable-${packageVersion}.txt) \
       --parallel=${GINKGO_PARALLEL} \
-      --test-args="-storage.testdriver=${PWD}/manifests.yaml -kubeconfig=$KUBECONFIG -node-os-distro=${NODE_OS_DISTRO}"
-
+      --test-args="-storage.testdriver=${PWD}/manifests.yaml -kubeconfig=${KUBECONFIG} -node-os-distro=${NODE_OS_DISTRO}"
     TEST_PASSED=$?
     set -e
     set +x
@@ -260,68 +155,67 @@ else
   fi
 
   if [[ $TEST_PATH == "./tests/e2e/..." ]]; then
-    eval "EXPANDED_TEST_EXTRA_FLAGS=$TEST_EXTRA_FLAGS"
     set -x
     set +e
-    ${GINKGO_BIN} -p -nodes="${GINKGO_NODES}" -v --focus="${GINKGO_FOCUS}" --skip="${GINKGO_SKIP}" "${TEST_PATH}" -- -kubeconfig="${KUBECONFIG}" -report-dir="${ARTIFACTS}" -gce-zone="${FIRST_ZONE}" "${EXPANDED_TEST_EXTRA_FLAGS}"
+    "${BIN}/ginkgo" -p -nodes="${GINKGO_PARALLEL}" -v \
+      --focus="${GINKGO_FOCUS}" \
+      --skip="${GINKGO_SKIP}" \
+      "${BASE_DIR}/../../tests/e2e/..." \
+      -- \
+      -kubeconfig="${KUBECONFIG}" \
+      -report-dir="${TEST_DIR}/artifacts" \
+      -gce-zone="${FIRST_ZONE}"
     TEST_PASSED=$?
     set -e
     set +x
   fi
 
-  PODS=$(kubectl get pod -n kube-system -l "app.kubernetes.io/name=${DRIVER_NAME},app.kubernetes.io/instance=${DRIVER_NAME}" -o json --kubeconfig "${KUBECONFIG}" | jq -r .items[].metadata.name)
+  PODS=$(kubectl get pod -n kube-system -l "app.kubernetes.io/name=aws-ebs-csi-driver,app.kubernetes.io/instance=aws-ebs-csi-driver" -o json --kubeconfig "${KUBECONFIG}" | jq -r .items[].metadata.name)
 
   while IFS= read -r POD; do
-    loudecho "Printing pod ${POD} ${CONTAINER_NAME} container logs"
+    loudecho "Printing pod ${POD} container logs"
     set +e
-    kubectl logs "${POD}" -n kube-system "${CONTAINER_NAME}" \
-      --kubeconfig "${KUBECONFIG}"
+    kubectl logs "${POD}" -n kube-system --all-containers --ignore-errors --kubeconfig "${KUBECONFIG}"
     set -e
-  done <<< "${PODS}"
+  done <<<"${PODS}"
 fi
 
-OVERALL_TEST_PASSED="${TEST_PASSED}"
-
 if [[ "${COLLECT_METRICS}" == true ]]; then
   metrics_collector "$KUBECONFIG" \
     "$AWS_ACCOUNT_ID" \
     "$AWS_REGION" \
     "$NODE_OS_DISTRO" \
     "$deployTimeSeconds" \
-    "$DRIVER_NAME" \
+    "aws-ebs-csi-driver" \
     "$VERSION"
 fi
 
-if [[ "${CLEAN}" == true ]]; then
-  loudecho "Cleaning"
+## Cleanup
 
-  if [[ "${HELM_CT_TEST}" != true ]]; then
-    loudecho "Removing driver via ${DEPLOY_METHOD}"
-    if [[ ${DEPLOY_METHOD} == "helm" ]]; then
-      ${HELM_BIN} del "${DRIVER_NAME}" \
-        --namespace kube-system \
-        --kubeconfig "${KUBECONFIG}"
-    elif [[ ${DEPLOY_METHOD} == "kustomize" ]]; then
-      kubectl --kubeconfig "${KUBECONFIG}" delete -k "./deploy/kubernetes/overlays/stable"
-    fi
+if [[ "${HELM_CT_TEST}" != true ]]; then
+  loudecho "Removing driver via ${DEPLOY_METHOD}"
+  if [[ ${DEPLOY_METHOD} == "helm" ]]; then
+    ${BIN}/helm del "aws-ebs-csi-driver" \
+      --namespace kube-system \
+      --kubeconfig "${KUBECONFIG}"
+  elif [[ ${DEPLOY_METHOD} == "kustomize" ]]; then
+    kubectl --kubeconfig "${KUBECONFIG}" delete -k "${BASE_DIR}/../../deploy/kubernetes/overlays/stable"
   fi
+fi
 
-  if [[ "${CLUSTER_TYPE}" == "kops" ]]; then
-    kops_delete_cluster \
-      "${KOPS_BIN}" \
-      "${CLUSTER_NAME}" \
-      "${KOPS_STATE_FILE}"
-  elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
-    eksctl_delete_cluster \
-      "${EKSCTL_BIN}" \
-      "${CLUSTER_NAME}"
-  fi
-else
-  loudecho "Not cleaning"
+if [[ "${EBS_INSTALL_SNAPSHOT}" == true ]]; then
+  loudecho "Installing snapshot controller and CRDs"
+  kubectl delete --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml
+  kubectl delete --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml
+  kubectl delete --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml
+  kubectl delete --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml
+  kubectl delete --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml
 fi
 
-loudecho "OVERALL_TEST_PASSED: ${OVERALL_TEST_PASSED}"
-if [[ $OVERALL_TEST_PASSED -ne 0 ]]; then
+## Output result
+
+loudecho "TEST_PASSED: ${TEST_PASSED}"
+if [[ $TEST_PASSED -ne 0 ]]; then
   loudecho "FAIL!"
   exit 1
 else
diff --git a/hack/provenance b/hack/provenance.sh
similarity index 96%
rename from hack/provenance
rename to hack/provenance.sh
index c04e7ebee..fcbc141c8 100755
--- a/hack/provenance
+++ b/hack/provenance.sh
@@ -26,7 +26,7 @@
 # Thus, this script echos back the flag `--provenance=false` if and only
 # if the local buildx installation supports it. If not, it exits silently.
 
-BUILDX_TEST=`docker buildx build --provenance=false 2>&1`
+BUILDX_TEST=$(docker buildx build --provenance=false 2>&1)
 if [[ "${BUILDX_TEST}" == *"See 'docker buildx build --help'."* ]]; then
   if [[ "${BUILDX_TEST}" == *"requires exactly 1 argument"* ]] && ! docker buildx inspect | grep -qE "^Driver:\s*docker$"; then
     echo "--provenance=false"
diff --git a/hack/prow-e2e.sh b/hack/prow-e2e.sh
new file mode 100755
index 000000000..57a13b110
--- /dev/null
+++ b/hack/prow-e2e.sh
@@ -0,0 +1,74 @@
+#!/bin/bash
+
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script runs tests in CI by creating a cluster, running the tests,
+# cleaning up (regardless of test success/failure), and passing out the result
+
+case ${1} in
+test-e2e-single-az)
+  TEST="single-az"
+  export AWS_AVAILABILITY_ZONES="us-west-2a"
+  ;;
+test-e2e-multi-az)
+  TEST="multi-az"
+  ;;
+test-e2e-external)
+  TEST="external"
+  ;;
+test-e2e-external-arm64)
+  TEST="external-arm64"
+  export INSTANCE_TYPE="m7g.medium"
+  export AMI_PARAMETER="/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64"
+  ;;
+test-e2e-external-eks)
+  TEST="external"
+  export CLUSTER_TYPE="eksctl"
+  ;;
+test-e2e-external-eks-windows)
+  TEST="external-windows"
+  export CLUSTER_TYPE="eksctl"
+  export WINDOWS="true"
+  ;;
+test-e2e-external-kustomize)
+  TEST="external-kustomize"
+  ;;
+test-helm-chart)
+  TEST="helm-ct"
+  ;;
+*)
+  echo "Unknown e2e test ${1}" >&2
+  exit 1
+  ;;
+esac
+
+export CLUSTER_NAME="ebs-csi-e2e-${RANDOM}.k8s.local"
+# Use S3 bucket created for CI
+export KOPS_BUCKET=${KOPS_BUCKET:-"k8s-kops-csi-shared-e2e"}
+# Always use us-west-2 in CI, no matter where the local client is
+export AWS_REGION=us-west-2
+
+make cluster/create || exit 1
+make e2e/${TEST}
+E2E_PASSED=$?
+make cluster/delete
+
+echo "E2E_PASSED: ${E2E_PASSED}"
+if [[ $E2E_PASSED -ne 0 ]]; then
+  echo "FAIL!"
+  exit 1
+else
+  echo "SUCCESS!"
+fi
diff --git a/hack/prow.sh b/hack/prow.sh
index a5fd118ef..0cf0d778d 100755
--- a/hack/prow.sh
+++ b/hack/prow.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2019 The Kubernetes Authors.
+# Copyright 2023 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -42,4 +42,4 @@ loudecho "Push manifest list containing amazon linux and windows based images to
 export REGISTRY=$REGISTRY_NAME
 export TAG=$GIT_TAG
 export VERSION=$PULL_BASE_REF
-IMAGE=gcr.io/k8s-staging-provider-aws/aws-ebs-csi-driver make all-push
+IMAGE=gcr.io/k8s-staging-provider-aws/aws-ebs-csi-driver make -j $(nproc) all-push-with-a1compat
diff --git a/hack/release b/hack/release
deleted file mode 100755
index 26ffa1d0f..000000000
--- a/hack/release
+++ /dev/null
@@ -1,134 +0,0 @@
-#!/usr/local/bin/python3
-
-# Copyright 2019 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import argparse
-import hashlib
-import json
-import os
-import requests
-
-def file_sha512(fileName, repoName):
-    download(fileName, repoName)
-    with open(fileName, 'rb') as file:
-        m = hashlib.sha512()
-        blob = file.read()
-        m.update(blob)
-        print("[{}](https://github.com/{}/archive/{}) | `{}`"
-                .format(fileName,repoName, fileName,m.hexdigest()))
-    os.remove(fileName)
-
-def download(fileName, repoName):
-    url = 'https://github.com/{}/archive/{}'.format(repoName, fileName)
-    r = requests.get(url, allow_redirects=True)
-    open(fileName, 'wb').write(r.content)
-
-def print_header(repo, version):
-    # Title
-    print('# {}'.format(version))
-
-    # documentation section
-    print('[Documentation](https://github.com/{}/blob/{}/docs/README.md)\n'
-            .format(repo,version))
-
-    # sha512
-    print('filename  | sha512 hash')
-    print('--------- | ------------')
-    file_sha512(version+".zip", repo)
-    file_sha512(version+".tar.gz", repo)
-
-class Github:
-    def __init__(self, user, token):
-        self._url = 'https://api.github.com'
-        self._user = user
-        self._token = token
-
-    def get_commits(self, repo, since, branch):
-        resp = requests.get('{}/repos/{}/compare/{}...{}'.format(self._url, repo, since, branch),
-                auth=(self._user, self._token))
-        jsonResp = json.loads(resp.content)
-        return jsonResp['commits']
-
-    def to_pr_numbers(self, repo, commit):
-        sha = commit['sha']
-        resp = requests.get('{}/repos/{}/commits/{}/pulls'.format(self._url, repo, sha),
-                headers={'Accept': 'application/vnd.github.groot-preview+json'},
-                auth=(self._user, self._token))
-        jsonResp = json.loads(resp.content)
-        ret = []
-        for pr in jsonResp:
-            ret.append(pr['number'])
-
-        return ret
-
-    def get_pr(self, repo, pr_number):
-        resp = requests.get('{}/repos/{}/pulls/{}'.format(self._url, repo, pr_number),
-                auth=(self._user, self._token))
-        jsonResp = json.loads(resp.content)
-        return jsonResp
-
-    def print_release_note(self, repo, since, branch):
-        # remove merge commits
-        commits = self.get_commits(repo, since, branch)
-        commits = filter(lambda c: not c['commit']['message'].startswith('Merge pull request'), commits)
-        pr_numbers = set()
-        for commit in commits:
-            numbers = self.to_pr_numbers(repo, commit)
-            for pr in numbers:
-                pr_numbers.add(pr)
-
-        # dedupe pr numbers
-        pr_numbers = sorted(list(pr_numbers))
-
-        for number in pr_numbers:
-            pr = self.get_pr(repo, number)
-            if 'user' in pr:
-                user = pr['user']['login']
-                print('* {} ([#{}]({}), [@{}](https://github.com/{}))'.format(pr['title'], pr['number'], pr['html_url'], user, user))
-
-def print_sha(args):
-    version = args.version
-    repo = args.repo
-    print_header(repo, version)
-
-def print_notes(args):
-    repo = args.repo
-    since = args.since
-    branch = args.branch
-    user = args.github_user
-    token = args.github_token
-
-    g = Github(user, token)
-    g.print_release_note(repo, since, branch)
-
-if __name__=="__main__":
-    parser = argparse.ArgumentParser(description='Generate release CHANGELOG')
-    parser.add_argument('--repo', metavar='repo', type=str, default='kubernetes-sigs/aws-ebs-csi-driver', help='the full github repository name')
-    parser.add_argument('--github-user', metavar='user', type=str, help='the github user for github api')
-    parser.add_argument('--github-token', metavar='token', type=str, help='the github token for github api')
-
-    subParsers = parser.add_subparsers(title='subcommands', description='[note|sha]')
-
-    noteParser = subParsers.add_parser('note', help='generate release notes')
-    noteParser.add_argument('--since', metavar='since', type=str, required=True, help='since version tag, e.g. if releasing v1.3.5 then set this to v1.3.4')
-    noteParser.add_argument('--branch', metavar='branch', type=str, required=True, help='release branch, e.g. if releasing v1.3.5 then set this to release-1.3')
-    noteParser.set_defaults(func=print_notes)
-
-    shaParser = subParsers.add_parser('sha', help='generate SHA for released version tag')
-    shaParser.add_argument('--version', metavar='version', type=str, required=True, help='the version to release')
-    shaParser.set_defaults(func=print_sha)
-
-    args = parser.parse_args()
-    args.func(args)
diff --git a/hack/release-scripts/generate-release-pr b/hack/release-scripts/generate-release-pr
index 89f98f3d4..11c3ceea8 100755
--- a/hack/release-scripts/generate-release-pr
+++ b/hack/release-scripts/generate-release-pr
@@ -38,7 +38,7 @@ check_dependencies() {
   if [[ $(uname) = "Darwin" ]]; then
     if ! command -v "gsed" &>/dev/null; then
       log "gsed could not be found, please install it."
-        exit 1
+      exit 1
     fi
     SED="gsed"
   fi
@@ -52,13 +52,13 @@ error_handler() {
 trap 'error_handler ${LINENO} $? "$BASH_COMMAND"' ERR
 
 # --- Script
-usage () {
+usage() {
   echo "Usage: $0 [PREV_DRIVER_VERSION] [NEW_DRIVER_VERSION]"
   echo "example: $0 v1.23.1 v1.24.0"
   exit 1
 }
 
-setup_vars () {
+setup_vars() {
   export PREV_DRIVER_VERSION=$1
   export NEW_DRIVER_VERSION=$2
 
@@ -72,18 +72,21 @@ setup_vars () {
   export CHART_PATH="$ROOT_DIRECTORY/charts/aws-ebs-csi-driver/Chart.yaml"
 }
 
-parse_args () {
+parse_args() {
   # Confirm 2 parameters
   [[ $# -ne 2 ]] && usage
 
   # Confirm new driver version > prev driver version
   log "Confirming $1 < $2"
-  sort -C -V <(echo "$1"; echo "$2") || usage
+  sort -C -V <(
+    echo "$1"
+    echo "$2"
+  ) || usage
 
   setup_vars "$@"
 }
 
-update_readme () {
+update_readme() {
   log "Updating README.md"
   # vi macro that adds new driver version 'Container Images' row to README.md
   vi -s <(echo "gg/## Container Images
@@ -91,36 +94,39 @@ update_readme () {
   jjjjjjp:wq") "$README_PATH"
 }
 
-update_makefile () {
+update_makefile() {
   log "Updating Makefile"
   $SED "s/VERSION?=$PREV_DRIVER_VERSION/VERSION?=$NEW_DRIVER_VERSION/g" -i "$MAKEFILE_PATH"
 }
 
-update_installmd () {
+update_installmd() {
   log "Updating docs/install.md"
   prev_major_minor_version=$(echo "$PREV_DRIVER_VERSION" | sed 's/v\([0-9]*\.[0-9]*\).*/\1/')
   new_major_minor_version=$(echo "$NEW_DRIVER_VERSION" | sed 's/v\([0-9]*\.[0-9]*\).*/\1/')
   $SED "s/?ref=release-$prev_major_minor_version/?ref=release-$new_major_minor_version/g" -i "$INSTALL_MD_PATH"
 }
 
-update_chart_and_overlays () {
+update_chart_and_overlays() {
   log "Updating helm chart and generates kustomize"
   prev_minor_patch_version=$(echo "$PREV_DRIVER_VERSION" | sed 's/v[0-9]*\.//')
   new_minor_patch_version=$(echo "$NEW_DRIVER_VERSION" | sed 's/v[0-9]*\.//')
 
   $SED "s/$prev_minor_patch_version$/$new_minor_patch_version/g" -i "$CHART_PATH"
 
-  (cd "$ROOT_DIRECTORY"; make generate-kustomize) > "/dev/null"
+  (
+    cd "$ROOT_DIRECTORY"
+    make generate-kustomize
+  ) >"/dev/null"
 }
 
-update_upstream_repo () {
+update_upstream_repo() {
   update_readme
   update_makefile
   update_installmd
   update_chart_and_overlays
 }
 
-print_rest_of_release_steps () {
+print_rest_of_release_steps() {
   echo "SUCCESS!
 Before you submit the release PR, you must also:
   1. Check that 'git diff' produces what you expected.
@@ -128,7 +134,7 @@ Before you submit the release PR, you must also:
   3. Update charts/aws-ebs-csi-driver/CHANGELOG.md"
 }
 
-main () {
+main() {
   check_dependencies
   parse_args "$@"
 
diff --git a/hack/release-scripts/generate-sidecar-tags b/hack/release-scripts/generate-sidecar-tags
index 5062658e4..8143ea12d 100755
--- a/hack/release-scripts/generate-sidecar-tags
+++ b/hack/release-scripts/generate-sidecar-tags
@@ -50,7 +50,7 @@ check_dependencies() {
   if [[ $(uname) = "Darwin" ]]; then
     if ! command -v "gsed" &>/dev/null; then
       log "gsed could not be found, please install it."
-        exit 1
+      exit 1
     fi
     SED="gsed"
   fi
@@ -66,7 +66,7 @@ trap 'error_handler ${LINENO} $? "$BASH_COMMAND"' ERR
 # --- Script
 trap 'rm $tmp_filename' EXIT
 
-update_gcr_kustomize_sidecar_tag () {
+update_gcr_kustomize_sidecar_tag() {
   sidecar_name=$1
   line_above=$2
 
@@ -75,7 +75,7 @@ update_gcr_kustomize_sidecar_tag () {
   $SED -i "\|$line_above|{n;s/.*/    newTag: $tag/;}" "$KUSTOMIZE_FILEPATH"
 }
 
-update_helm_chart_sidecar_tag () {
+update_helm_chart_sidecar_tag() {
   sidecar_name=$1
 
   export TAG
@@ -84,7 +84,7 @@ update_helm_chart_sidecar_tag () {
   yq ".sidecars.$sidecar_name.image.tag = env(TAG)" -i "$HELM_VALUES_FILEPATH"
 }
 
-generate_gcr_kustomize () {
+generate_gcr_kustomize() {
   update_gcr_kustomize_sidecar_tag "provisioner" "newName: registry.k8s.io/sig-storage/csi-provisioner"
   update_gcr_kustomize_sidecar_tag "attacher" "newName: registry.k8s.io/sig-storage/csi-attacher"
   update_gcr_kustomize_sidecar_tag "livenessProbe" "newName: registry.k8s.io/sig-storage/livenessprobe"
@@ -95,18 +95,17 @@ generate_gcr_kustomize () {
   log "Success: All sidecar tags in $KUSTOMIZE_FILEPATH updated"
 }
 
-generate_helm_sidecars () {
-  yq '.sidecars | keys | .[]' "$IMAGE_DIGESTS_FILEPATH" > "$tmp_filename"
+generate_helm_sidecars() {
+  yq '.sidecars | keys | .[]' "$IMAGE_DIGESTS_FILEPATH" >"$tmp_filename"
 
-  for sidecar in $(cat "$tmp_filename")
-     do
-       update_helm_chart_sidecar_tag "$sidecar"
-     done
+  for sidecar in $(cat "$tmp_filename"); do
+    update_helm_chart_sidecar_tag "$sidecar"
+  done
 
   log "Success: All sidecar tags in $HELM_VALUES_FILEPATH updated"
 }
 
-main () {
+main() {
   check_dependencies
   generate_gcr_kustomize
   generate_helm_sidecars
diff --git a/hack/release-scripts/get-latest-sidecar-images b/hack/release-scripts/get-latest-sidecar-images
index 7a7428a12..6de372f24 100755
--- a/hack/release-scripts/get-latest-sidecar-images
+++ b/hack/release-scripts/get-latest-sidecar-images
@@ -52,7 +52,7 @@ trap 'error_handler ${LINENO} $? "$BASH_COMMAND"' ERR
 # --- Script
 trap 'rm $tmp_filename' EXIT
 
-generate_image_digests_file () {
+generate_image_digests_file() {
   touch "$OUTPUT_FILEPATH"
 
   yq '.sidecars.snapshotter.image = "public.ecr.aws/eks-distro/kubernetes-csi/external-snapshotter/csi-snapshotter"' -i "$OUTPUT_FILEPATH"
@@ -68,28 +68,27 @@ crane_get_latest_image_tag() {
   image=$1
 
   export TAG
-  TAG=$(crane ls "$image" | sed '/latest/d' | sort -V | tail -1)  # Get tag for $image with latest semvar
+  TAG=$(crane ls "$image" | sed '/latest/d' | sort -V | tail -1) # Get tag for $image with latest semvar
 }
 
-update_sidecars_source_of_truth () {
-  yq '.sidecars | keys | .[]' "$OUTPUT_FILEPATH" > "$tmp_filename"
+update_sidecars_source_of_truth() {
+  yq '.sidecars | keys | .[]' "$OUTPUT_FILEPATH" >"$tmp_filename"
 
-  for sidecar in $(cat "$tmp_filename")
-     do
-       log "Updating $sidecar in $OUTPUT_FILEPATH"
-       image=$(yq ".sidecars.$sidecar.image" "$OUTPUT_FILEPATH")
+  for sidecar in $(cat "$tmp_filename"); do
+    log "Updating $sidecar in $OUTPUT_FILEPATH"
+    image=$(yq ".sidecars.$sidecar.image" "$OUTPUT_FILEPATH")
 
-       export TAG
-       crane_get_latest_image_tag "$image"
-       yq ".sidecars.$sidecar.tag = env(TAG)" -i "$OUTPUT_FILEPATH"
+    export TAG
+    crane_get_latest_image_tag "$image"
+    yq ".sidecars.$sidecar.tag = env(TAG)" -i "$OUTPUT_FILEPATH"
 
-       export DIGEST
-       DIGEST=$(crane digest "$image:$TAG")
-       yq ".sidecars.$sidecar.manifestDigest = env(DIGEST)" -i "$OUTPUT_FILEPATH"
-     done
+    export DIGEST
+    DIGEST=$(crane digest "$image:$TAG")
+    yq ".sidecars.$sidecar.manifestDigest = env(DIGEST)" -i "$OUTPUT_FILEPATH"
+  done
 }
 
-main () {
+main() {
   check_dependencies
   generate_image_digests_file
   update_sidecars_source_of_truth
diff --git a/hack/test-integration.sh b/hack/test-integration.sh
deleted file mode 100755
index 906e551df..000000000
--- a/hack/test-integration.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/bin/bash
-
-# Copyright 2019 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -euo pipefail
-
-if ! [[ "$0" =~ hack/test-integration.sh ]]; then
-  echo "must be run from repository root"
-  exit 127
-fi
-
-export GO111MODULE=on
-go test -c ./tests/integration/... -o bin/integration.test && \
-  sudo -E bin/integration.test -test.v -ginkgo.v
diff --git a/hack/tools/install.sh b/hack/tools/install.sh
new file mode 100755
index 000000000..a4ddd05d6
--- /dev/null
+++ b/hack/tools/install.sh
@@ -0,0 +1,183 @@
+#!/bin/bash
+
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+# https://pypi.org/project/awscli/
+AWSCLI_VERSION="1.31.7"
+# https://github.com/helm/chart-testing
+CT_VERSION="v3.10.1"
+# https://github.com/eksctl-io/eksctl
+EKSCTL_VERSION="v0.165.0"
+# https://github.com/onsi/ginkgo
+GINKGO_VERSION="v2.13.2"
+# https://github.com/golangci/golangci-lint
+GOLANGCI_LINT_VERSION="v1.54.0"
+# https://github.com/helm/helm
+HELM_VERSION="v3.13.2"
+# https://github.com/kubernetes/kops
+KOPS_VERSION="v1.28.1"
+# https://github.com/golang/mock
+MOCKGEN_VERSION="v1.6.0"
+# https://github.com/patrickvane/shfmt
+SHFMT_VERSION="v3.7.0"
+# https://pypi.org/project/yamale/
+YAMALE_VERSION="4.0.4"
+# https://pypi.org/project/yamllint/
+YAMLLINT_VERSION="1.32.0"
+
+OS="$(go env GOHOSTOS)"
+ARCH="$(go env GOHOSTARCH)"
+
+# Installation helpers
+
+function install_binary() {
+  INSTALL_PATH="${1}"
+  DOWNLOAD_URL="${2}"
+  BINARY_NAME="${3}"
+
+  curl --location "${DOWNLOAD_URL}" --output "${INSTALL_PATH}/${BINARY_NAME}"
+  chmod +x "${INSTALL_PATH}/${BINARY_NAME}"
+}
+
+function install_go() {
+  INSTALL_PATH="${1}"
+  PACKAGE="${2}"
+
+  export GOBIN="${INSTALL_PATH}"
+  go install "${PACKAGE}"
+}
+
+function install_pip() {
+  INSTALL_PATH="${1}"
+  PACKAGE="${2}"
+  COMMAND="${3}"
+
+  source "${INSTALL_PATH}/venv/bin/activate"
+  python3 -m pip install "${PACKAGE}"
+  cp "$(dirname "${0}")/python-runner.sh" "${INSTALL_PATH}/${COMMAND}"
+}
+
+function install_tar_binary() {
+  INSTALL_PATH="${1}"
+  DOWNLOAD_URL="${2}"
+  BINARY_PATH="${3}"
+
+  BINARY_NAME="$(basename "${BINARY_PATH}")"
+
+  if [ "${DOWNLOAD_URL##*.}" = "gz" ]; then
+    TAR_EXTRA_FLAGS="-z"
+  elif [ "${DOWNLOAD_URL##*.}" = "xz" ]; then
+    TAR_EXTRA_FLAGS="-J"
+  else
+    TAR_EXTRA_FLAGS=""
+  fi
+
+  curl --location "${DOWNLOAD_URL}" | tar "$TAR_EXTRA_FLAGS" --extract --touch --transform "s/.*/${BINARY_NAME}/" -C "${INSTALL_PATH}" "${BINARY_PATH}"
+  chmod +x "${INSTALL_PATH}/${BINARY_NAME}"
+}
+
+# Tool-specific installers
+
+function install_aws() {
+  INSTALL_PATH="${1}"
+
+  install_pip "${INSTALL_PATH}" "awscli==${AWSCLI_VERSION}" "aws"
+}
+
+function install_ct() {
+  INSTALL_PATH="${1}"
+
+  install_tar_binary "${INSTALL_PATH}" "https://github.com/helm/chart-testing/releases/download/${CT_VERSION}/chart-testing_${CT_VERSION:1}_${OS}_${ARCH}.tar.gz" "ct"
+  install_pip "${INSTALL_PATH}" "yamale==${YAMALE_VERSION}" "yamale"
+  install_pip "${INSTALL_PATH}" "yamllint==${YAMLLINT_VERSION}" "yamllint"
+}
+
+function install_eksctl() {
+  INSTALL_PATH="${1}"
+
+  install_tar_binary "${INSTALL_PATH}" "https://github.com/weaveworks/eksctl/releases/download/${EKSCTL_VERSION}/eksctl_${OS^}_${ARCH}.tar.gz" "eksctl"
+}
+
+function install_ginkgo() {
+  INSTALL_PATH="${1}"
+
+  install_go "${INSTALL_PATH}" "github.com/onsi/ginkgo/v2/ginkgo@${GINKGO_VERSION}"
+}
+
+function install_golangci-lint() {
+  INSTALL_PATH="${1}"
+
+  # golangci-lint recommends against installing with `go install`: https://golangci-lint.run/usage/install/#install-from-source
+  install_tar_binary "${INSTALL_PATH}" "https://github.com/golangci/golangci-lint/releases/download/${GOLANGCI_LINT_VERSION}/golangci-lint-${GOLANGCI_LINT_VERSION:1}-${OS}-${ARCH}.tar.gz" "golangci-lint-${GOLANGCI_LINT_VERSION:1}-${OS}-${ARCH}/golangci-lint"
+}
+
+function install_helm() {
+  INSTALL_PATH="${1}"
+
+  install_tar_binary "${INSTALL_PATH}" "https://get.helm.sh/helm-${HELM_VERSION}-${OS}-${ARCH}.tar.gz" "${OS}-${ARCH}/helm"
+}
+
+function install_kops() {
+  INSTALL_PATH="${1}"
+
+  install_binary "${INSTALL_PATH}" "https://github.com/kubernetes/kops/releases/download/${KOPS_VERSION}/kops-${OS}-${ARCH}" "kops"
+}
+
+function install_kubetest2() {
+  INSTALL_PATH="${1}"
+
+  install_go "${INSTALL_PATH}" "sigs.k8s.io/kubetest2/...@latest"
+}
+
+function install_mockgen() {
+  INSTALL_PATH="${1}"
+
+  install_go "${INSTALL_PATH}" "github.com/golang/mock/mockgen@${MOCKGEN_VERSION}"
+}
+
+function install_shfmt() {
+  INSTALL_PATH="${1}"
+
+  install_go "${INSTALL_PATH}" "mvdan.cc/sh/v3/cmd/shfmt@${SHFMT_VERSION}"
+}
+
+# Utility functions
+
+function create_environment() {
+  INSTALL_PATH="${1}"
+
+  if command -v "python3"; then
+    PYTHON_CMD="python3"
+  else
+    PYTHON_CMD="python"
+  fi
+  VIRTUAL_ENV_DISABLE_PROMPT=1 "${PYTHON_CMD}" -m venv "${INSTALL_PATH}/venv"
+}
+
+function install_tool() {
+  INSTALL_PATH="${1}"
+  TOOL="${2}"
+
+  "install_${TOOL}" "${INSTALL_PATH}"
+}
+
+# Script dispatcher
+
+if [ ! -d "${TOOLS_PATH}/venv" ]; then
+  create_environment "${TOOLS_PATH}"
+fi
+install_tool "${TOOLS_PATH}" "${1}"
diff --git a/hack/verify-all b/hack/tools/python-runner.sh
similarity index 69%
rename from hack/verify-all
rename to hack/tools/python-runner.sh
index e6cabf881..67c8434ea 100755
--- a/hack/verify-all
+++ b/hack/tools/python-runner.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2019 The Kubernetes Authors.
+# Copyright 2023 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,11 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -euo pipefail
+# This script is used as a stub for python commands installed to bin/
+# It activates the venv inside bin/venv/ and then passes through
 
-PKG_ROOT=$(git rev-parse --show-toplevel)
-
-${PKG_ROOT}/hack/verify-gofmt
-${PKG_ROOT}/hack/verify-govet
-${PKG_ROOT}/bin/golangci-lint run --deadline=10m
-${PKG_ROOT}/hack/verify-vendor.sh
+source "$(dirname "${0}")/venv/bin/activate"
+exec "$(basename "${0}")" "$@"
diff --git a/hack/update-gofmt b/hack/update-gofmt
deleted file mode 100755
index dd016e176..000000000
--- a/hack/update-gofmt
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-
-# Copyright 2019 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -euo pipefail
-
-find . -name "*.go" | grep -v "\/vendor\/" | xargs gofmt -s -w
diff --git a/hack/update-gomock b/hack/update-gomock
deleted file mode 100755
index ee6600dda..000000000
--- a/hack/update-gomock
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-
-# Copyright 2019 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -euo pipefail
-
-mockgen -package cloud -destination=./pkg/cloud/mock_cloud.go -source pkg/cloud/cloud_interface.go
-mockgen -package cloud -destination=./pkg/cloud/mock_metadata.go -source pkg/cloud/metadata_interface.go
-mockgen -package driver -destination=./pkg/driver/mock_mount.go -source pkg/driver/mount.go
-mockgen -package mounter -destination=./pkg/mounter/mock_mount_windows.go -source pkg/mounter/safe_mounter_windows.go
-
-# Reflection-based mocking for external dependencies
-mockgen -package cloud -destination=./pkg/cloud/mock_ec2.go github.com/aws/aws-sdk-go/service/ec2/ec2iface EC2API
-mockgen -package driver -destination=./pkg/driver/mock_k8s_client.go -mock_names='Interface=MockKubernetesClient' k8s.io/client-go/kubernetes Interface
-mockgen -package driver -destination=./pkg/driver/mock_k8s_corev1.go k8s.io/client-go/kubernetes/typed/core/v1 CoreV1Interface,NodeInterface
-mockgen -package driver -destination=./pkg/driver/mock_k8s_storagev1.go k8s.io/client-go/kubernetes/typed/storage/v1 VolumeAttachmentInterface,StorageV1Interface
-
-# Fixes "Mounter Type cannot implement 'Mounter' as it has a non-exported method and is defined in a different package"
-# See https://github.com/kubernetes/mount-utils/commit/a20fcfb15a701977d086330b47b7efad51eb608e for context.
-sed -i '/type MockMounter struct {/a \\tmount_utils.Interface' pkg/driver/mock_mount.go
-sed -i '/type MockProxyMounter struct {/a \\tmount.Interface' pkg/mounter/mock_mount_windows.go
diff --git a/hack/update-gomod b/hack/update-gomod
deleted file mode 100755
index d89c2a7ed..000000000
--- a/hack/update-gomod
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/bin/bash
-
-# Copyright 2019 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-set -euo pipefail
-set -x
-
-VERSION=${1#"v"}
-if [ -z "$VERSION" ]; then
-    echo "Must specify version!"
-    exit 1
-fi
-MODS=($(
-    curl -sS https://raw.githubusercontent.com/kubernetes/kubernetes/v${VERSION}/go.mod |
-    sed -n 's|.*k8s.io/\(.*\) => ./staging/src/k8s.io/.*|k8s.io/\1|p'
-))
-echo $MODS
-for MOD in "${MODS[@]}"; do
-    echo $MOD
-    V=$(
-        go mod download -json "${MOD}@kubernetes-${VERSION}" |
-        sed -n 's|.*"Version": "\(.*\)".*|\1|p'
-    )
-    go mod edit "-replace=${MOD}=${MOD}@${V}"
-done
diff --git a/hack/update-kustomize.sh b/hack/update-kustomize.sh
new file mode 100755
index 000000000..03f1d5b8d
--- /dev/null
+++ b/hack/update-kustomize.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script updates the kustomize templates in deploy/kubernetes/base/ by
+# running `helm template` and stripping the namespace from the output
+
+set -euo pipefail
+
+BIN="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/../bin"
+TEMP_DIR=$(mktemp -d)
+trap "rm -rf \"${TEMP_DIR}\"" EXIT
+cp "deploy/kubernetes/base/kustomization.yaml" "${TEMP_DIR}/kustomization.yaml"
+
+"${BIN}/helm" template --output-dir "${TEMP_DIR}" --skip-tests --api-versions 'snapshot.storage.k8s.io/v1' --api-versions 'policy/v1/PodDisruptionBudget' --set 'controller.userAgentExtra=kustomize' kustomize charts/aws-ebs-csi-driver >/dev/null
+rm -rf "deploy/kubernetes/base"
+mv "${TEMP_DIR}/aws-ebs-csi-driver/templates" "deploy/kubernetes/base"
+
+sed -i '/namespace:/d' deploy/kubernetes/base/*
+cp "${TEMP_DIR}/kustomization.yaml" "deploy/kubernetes/base/kustomization.yaml"
diff --git a/hack/update-mockgen.sh b/hack/update-mockgen.sh
new file mode 100755
index 000000000..5681f223f
--- /dev/null
+++ b/hack/update-mockgen.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+BIN="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/../bin"
+
+# Source-based mocking for internal interfaces
+"${BIN}/mockgen" -package cloud -destination=./pkg/cloud/mock_cloud.go -source pkg/cloud/cloud_interface.go
+"${BIN}/mockgen" -package cloud -destination=./pkg/cloud/mock_metadata.go -source pkg/cloud/metadata_interface.go
+"${BIN}/mockgen" -package driver -destination=./pkg/driver/mock_mount.go -source pkg/driver/mount.go
+"${BIN}/mockgen" -package mounter -destination=./pkg/mounter/mock_mount_windows.go -source pkg/mounter/safe_mounter_windows.go
+
+# Reflection-based mocking for external dependencies
+"${BIN}/mockgen" -package cloud -destination=./pkg/cloud/mock_ec2.go github.com/aws/aws-sdk-go/service/ec2/ec2iface EC2API
+"${BIN}/mockgen" -package driver -destination=./pkg/driver/mock_k8s_client.go -mock_names='Interface=MockKubernetesClient' k8s.io/client-go/kubernetes Interface
+"${BIN}/mockgen" -package driver -destination=./pkg/driver/mock_k8s_corev1.go k8s.io/client-go/kubernetes/typed/core/v1 CoreV1Interface,NodeInterface
+"${BIN}/mockgen" -package driver -destination=./pkg/driver/mock_k8s_storagev1.go k8s.io/client-go/kubernetes/typed/storage/v1 VolumeAttachmentInterface,StorageV1Interface
+
+# Fixes "Mounter Type cannot implement 'Mounter' as it has a non-exported method and is defined in a different package"
+# See https://github.com/kubernetes/mount-utils/commit/a20fcfb15a701977d086330b47b7efad51eb608e for context.
+sed -i '/type MockMounter struct {/a \\tmount_utils.Interface' pkg/driver/mock_mount.go
+sed -i '/type MockProxyMounter struct {/a \\tmount.Interface' pkg/mounter/mock_mount_windows.go
diff --git a/hack/verify-gofmt b/hack/verify-gofmt
deleted file mode 100755
index ae0106d65..000000000
--- a/hack/verify-gofmt
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/bin/bash
-
-# Copyright 2019 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -euo pipefail
-
-echo "Verifying gofmt"
-
-diff=$(find . -name "*.go" | grep -v "\/vendor\/" | xargs gofmt -s -d 2>&1)
-if [[ -n "${diff}" ]]; then
-  echo "${diff}"
-  echo
-  echo "Please run hack/update-gofmt to fix the issue(s)"
-  exit 1
-fi
-echo "No issue found"
diff --git a/hack/verify-govet b/hack/verify-govet
deleted file mode 100755
index f33dade33..000000000
--- a/hack/verify-govet
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash
-
-# Copyright 2019 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -euo pipefail
-
-echo "Verifying govet"
-
-go vet $(go list ./... | grep -v vendor)
-
-echo "Done"
diff --git a/hack/verify-kustomize b/hack/verify-update.sh
similarity index 50%
rename from hack/verify-kustomize
rename to hack/verify-update.sh
index ad4fb3cda..c6d1072ff 100755
--- a/hack/verify-kustomize
+++ b/hack/verify-update.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2022 The Kubernetes Authors.
+# Copyright 2023 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,23 +14,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# This script verifies that `make update` does not need to run
+# It does so by creating a temporary copy of the repo and running `make update`
+# in the copy, and then checking if it produces a diff to the local copy
+
 set -euo pipefail
 
-echo "Verifying kustomize"
-diff=$(git status --porcelain=v1)
-if [[ -n "${diff}" ]]; then
-  echo "${diff}"
-  echo
-  echo "Please commit all changes before verifying"
+ROOT="$(dirname "${0}")/../"
+TEST_DIR=$(mktemp -d)
+trap "rm -rf \"${TEST_DIR}\"" EXIT
+cp -rf "${ROOT}/." "${TEST_DIR}"
+
+if ! make -C "${TEST_DIR}" update >/dev/null; then
+  echo "\`make update\` failed!"
   exit 1
 fi
-make generate-kustomize
-echo
-diff=$(git status --porcelain=v1)
-if [[ -n "${diff}" ]]; then
-  echo "${diff}"
-  echo
-  echo "Please run make generate-kustomize to fix the issue(s)"
+
+if ! diff -r "${TEST_DIR}" "${ROOT}"; then
+  echo "Auto-generation/formatting needs to run!"
+  echo "Run \`make update\` to fix!"
   exit 1
 fi
-echo "No issue found"
diff --git a/hack/verify-vendor.sh b/hack/verify-vendor.sh
deleted file mode 100755
index e849dd46d..000000000
--- a/hack/verify-vendor.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2019 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-echo "Repo uses 'go mod'."
-if ! (set -x; env GO111MODULE=on go mod tidy); then
-    echo "ERROR: vendor check failed."
-    exit 1
-elif [ "$(git status --porcelain -- go.mod go.sum | wc -l)" -gt 0 ]; then
-    echo "ERROR: go module files *not* up-to-date, they did get modified by 'GO111MODULE=on go mod tidy':";
-    git diff -- go.mod go.sum
-    exit 1
-elif [ -d vendor ]; then
-    if ! (set -x; env GO111MODULE=on go mod vendor); then
-	echo "ERROR: vendor check failed."
-	exit 1
-    elif [ "$(git status --porcelain -- vendor | wc -l)" -gt 0 ]; then
-	echo "ERROR: vendor directory *not* up-to-date, it did get modified by 'GO111MODULE=on go mod vendor':"
-	git status -- vendor
-	git diff -- vendor
-	exit 1
-    else
-	echo "Go dependencies and vendor directory up-to-date."
-    fi
-else
-    echo "Go dependencies up-to-date."
-fi