🌱 Cleanup v1 cluster before creating v3 clusters on upgrade (#576)

* Cleanup v1 cluster before creating v3 clusters on upgrade

Signed-off-by: Danil-Grigorev <[email protected]>

* Cover provisioning to management v3 migration

- Add e2e tests, verifying the added migration functionality

Signed-off-by: Danil-Grigorev <[email protected]>

* Clean up space for short e2e run

Signed-off-by: Danil-Grigorev <[email protected]>

---------

Signed-off-by: Danil-Grigorev <[email protected]>

.github/workflows/e2e-short.yaml

@@ -8,6 +8,23 @@ jobs:
   e2e:
     runs-on: ubuntu-latest
     steps:
+      - name: Cleanup space
+        run: |
+          sudo apt-get remove -y '^aspnetcore-.*' || echo "::warning::The command [sudo apt-get remove -y '^aspnetcore-.*'] failed to complete successfully. Proceeding..."
+          sudo apt-get remove -y '^dotnet-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^dotnet-.*' --fix-missing] failed to complete successfully. Proceeding..."
+          sudo apt-get remove -y '^llvm-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^llvm-.*' --fix-missing] failed to complete successfully. Proceeding..."
+          sudo apt-get remove -y 'php.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y 'php.*' --fix-missing] failed to complete successfully. Proceeding..."
+          sudo apt-get remove -y '^mongodb-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^mongodb-.*' --fix-missing] failed to complete successfully. Proceeding..."
+          sudo apt-get remove -y '^mysql-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^mysql-.*' --fix-missing] failed to complete successfully. Proceeding..."
+          sudo apt-get remove -y azure-cli google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri --fix-missing || echo "::warning::The command [sudo apt-get remove -y azure-cli google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri --fix-missing] failed to complete successfully. Proceeding..."
+          sudo apt-get remove -y google-cloud-sdk --fix-missing || echo "::debug::The command [sudo apt-get remove -y google-cloud-sdk --fix-missing] failed to complete successfully. Proceeding..."
+          sudo apt-get remove -y google-cloud-cli --fix-missing || echo "::debug::The command [sudo apt-get remove -y google-cloud-cli --fix-missing] failed to complete successfully. Proceeding..."
+          sudo apt-get autoremove -y || echo "::warning::The command [sudo apt-get autoremove -y] failed to complete successfully. Proceeding..."
+          sudo apt-get clean || echo "::warning::The command [sudo apt-get clean] failed to complete successfully. Proceeding..."
+          sudo docker image prune --all --force || true
+          sudo swapoff -a || true
+          sudo rm -f /mnt/swapfile || true
+          free -h
       - name: Checkout
         uses: actions/[email protected]
         with:

charts/rancher-turtles/questions.yml

@@ -53,6 +53,12 @@ questions:
     type: boolean
     label: Use management v3 cluster manifest
     group: "Rancher Turtles Features Settings"
+  - variable: rancherTurtles.features.managementv3-cluster-migration.enabled
+    default: false
+    description: "(Experimental) Automatically migrate provisioning clusters to management clusters on upgrade"
+    type: boolean
+    label: Use newly provisioned management cluster manifest, replacing provisioning cluster manifest.
+    group: "Rancher Turtles Features Settings"
   - variable: cluster-api-operator.cluster-api.rke2.enabled
     default: "true"
     description: "Flag to enable or disable installation of the RKE2 provider for Cluster API. By default this is enabled."

charts/rancher-turtles/templates/post-upgrade-job.yaml

@@ -0,0 +1,67 @@
+{{- if and (index .Values "rancherTurtles" "features" "managementv3-cluster" "enabled") (index .Values "rancherTurtles" "features" "managementv3-cluster-migration" "enabled") }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: post-upgrade-job
+  namespace: rancher-turtles-system
+  annotations:
+    "helm.sh/hook": post-upgrade
+    "helm.sh/hook-weight": "1"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: post-upgrade-job-delete-clusters
+  annotations:
+    "helm.sh/hook": post-upgrade
+    "helm.sh/hook-weight": "1"
+rules:
+- apiGroups:
+  - provisioning.cattle.io
+  resources:
+  - clusters
+  verbs:
+  - list
+  - delete
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: post-upgrade-job-delete-clusters
+  annotations:
+    "helm.sh/hook": post-upgrade
+    "helm.sh/hook-weight": "1"
+subjects:
+  - kind: ServiceAccount
+    name: post-upgrade-job
+    namespace: rancher-turtles-system
+roleRef:
+  kind: ClusterRole
+  name: post-upgrade-job-delete-clusters
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: post-upgrade-delete-clusters
+  annotations:
+    "helm.sh/hook": post-upgrade
+    "helm.sh/hook-weight": "2"
+spec:
+  ttlSecondsAfterFinished: 300
+  template:
+    spec:
+      serviceAccountName: post-upgrade-job
+      containers:
+        - name: post-upgrade-delete-clusters
+          image: {{ index .Values "rancherTurtles" "features" "rancher-webhook" "kubectlImage" }}
+          args:
+          - delete
+          - clusters.provisioning.cattle.io
+          - --selector=cluster-api.cattle.io/owned
+          - -A
+          - --ignore-not-found=true
+          - --wait
+      restartPolicy: OnFailure
+{{- end }}

charts/rancher-turtles/values.yaml

@@ -18,6 +18,8 @@ rancherTurtles:
       label: true
     managementv3-cluster:
       enabled: false
+    managementv3-cluster-migration:
+      enabled: false
     propagate-labels:
       enabled: false
     etcd-snapshot-restore: