Implement SDN live migration

Signed-off-by: Peng Liu <[email protected]>
openshift · Dec 5, 2023 · 023da23 · 023da23
1 parent 2f20980
commit 023da23
Show file tree

Hide file tree

Showing 539 changed files with 57,980 additions and 514 deletions.
diff --git a/bindata/network/multus/multus.yaml b/bindata/network/multus/multus.yaml
@@ -143,6 +143,56 @@ data:
         "daemonSocketDir": "/run/multus/socket",
         "socketDir": "/host{{ .MultusSocketParentDir }}/socket"
     }
+{{- if .IsNetworkTypeLiveMigration}}
+  daemon-config-lm-ovn.json: |
+    {
+        "cniVersion": "0.3.1",
+        "chrootDir": "/hostroot",
+        "logToStderr": true,
+        "logLevel": "verbose",
+        "binDir": "{{ .CNIBinDir }}",
+{{ if .NETWORK_NODE_IDENTITY_ENABLE }}
+        "perNodeCertificate": {
+          "enabled": true,
+          "bootstrapKubeconfig": "{{ .KubeletKubeconfigPath }}",
+          "certDir": "/etc/cni/multus/certs",
+          "certDuration": "24h"
+        },
+{{ end }}
+        "cniConfigDir": "/host/etc/cni/net.d",
+        "multusConfigFile": "auto",
+        "multusAutoconfigDir": "/host/run/multus/cni/net.d",
+        "namespaceIsolation": true,
+        "globalNamespaces": "default,openshift-multus,openshift-sriov-network-operator",
+        "readinessindicatorfile": "/host/run/multus/cni/net.d/10-ovn-kubernetes.conf",
+        "daemonSocketDir": "/run/multus/socket",
+        "socketDir": "/host{{ .MultusSocketParentDir }}/socket"
+    }
+  daemon-config-lm-sdn.json: |
+    {
+        "cniVersion": "0.3.1",
+        "chrootDir": "/hostroot",
+        "logToStderr": true,
+        "logLevel": "verbose",
+        "binDir": "{{ .CNIBinDir }}",
+{{ if .NETWORK_NODE_IDENTITY_ENABLE }}
+        "perNodeCertificate": {
+          "enabled": true,
+          "bootstrapKubeconfig": "{{ .KubeletKubeconfigPath }}",
+          "certDir": "/etc/cni/multus/certs",
+          "certDuration": "24h"
+        },
+{{ end }}
+        "cniConfigDir": "/host/etc/cni/net.d",
+        "multusConfigFile": "auto",
+        "multusAutoconfigDir": "/host/run/multus/cni/net.d",
+        "namespaceIsolation": true,
+        "globalNamespaces": "default,openshift-multus,openshift-sriov-network-operator",
+        "readinessindicatorfile": "/host/run/multus/cni/net.d/80-openshift-network.conf",
+        "daemonSocketDir": "/run/multus/socket",
+        "socketDir": "/host{{ .MultusSocketParentDir }}/socket"
+    }
+{{- end}}
 ---
 kind: DaemonSet
 apiVersion: apps/v1
@@ -187,8 +237,16 @@ spec:
         command: [ "/bin/bash", "-ec", "--" ]
         args:
           - >
+            MULTUS_DAEMON_OPT=""
+{{ if .IsNetworkTypeLiveMigration }}
+            if ip link show br-ex; then
+              MULTUS_DAEMON_OPT="--config /etc/cni/net.d/multus.d/daemon-config-lm-ovn.json"
+            else
+              MULTUS_DAEMON_OPT="--config /etc/cni/net.d/multus.d/daemon-config-lm-sdn.json"
+            fi
+{{ end }}
             /entrypoint/cnibincopy.sh;
-            exec /usr/src/multus-cni/bin/multus-daemon
+            exec /usr/src/multus-cni/bin/multus-daemon $MULTUS_DAEMON_OPT
         resources:
           requests:
             cpu: 10m
@@ -322,6 +380,12 @@ spec:
             items:
             - key: daemon-config.json
               path: daemon-config.json
+{{ if .IsNetworkTypeLiveMigration }}
+            - key: daemon-config-lm-ovn.json
+              path: daemon-config-lm-ovn.json
+            - key: daemon-config-lm-sdn.json
+              path: daemon-config-lm-sdn.json
+{{ end }}
         - name: host-run-multus-certs
           hostPath:
             path: /etc/cni/multus/certs

diff --git a/bindata/network/node-identity/managed/node-identity.yaml b/bindata/network/node-identity/managed/node-identity.yaml
@@ -113,10 +113,9 @@ spec:
                 exit 1
               fi
             done
-            ho_enable=
-{{- if .OVNHybridOverlayEnable }}
+            # OVN-K will try to remove hybrid overlay node annotations even when the hybrid overlay is not enabled.
+            # https://github.com/ovn-org/ovn-kubernetes/blob/ac6820df0b338a246f10f412cd5ec903bd234694/go-controller/pkg/ovn/master.go#L791
             ho_enable="--enable-hybrid-overlay"
-{{ end }}
             echo "I$(date "+%m%d %H:%M:%S.%N") - network-node-identity - start webhook"
             # extra-allowed-user: service account `ovn-kubernetes-control-plane`
             # sets pod annotations in multi-homing layer3 network controller (cluster-manager)

diff --git a/bindata/network/node-identity/self-hosted/node-identity.yaml b/bindata/network/node-identity/self-hosted/node-identity.yaml
@@ -44,11 +44,9 @@ spec:
               source "/env/_master"
               set +o allexport
             fi
-
-            ho_enable=
-{{- if .OVNHybridOverlayEnable }}
+            # OVN-K will try to remove hybrid overlay node annotations even when the hybrid overlay is not enabled.
+            # https://github.com/ovn-org/ovn-kubernetes/blob/ac6820df0b338a246f10f412cd5ec903bd234694/go-controller/pkg/ovn/master.go#L791
             ho_enable="--enable-hybrid-overlay"
-{{ end }}
             echo "I$(date "+%m%d %H:%M:%S.%N") - network-node-identity - start webhook"
             # extra-allowed-user: service account `ovn-kubernetes-control-plane`
             # sets pod annotations in multi-homing layer3 network controller (cluster-manager)

diff --git a/bindata/network/openshift-sdn/002-rbac.yaml b/bindata/network/openshift-sdn/002-rbac.yaml
@@ -14,6 +14,13 @@ rules:
   - get
   - list
   - watch
+{{- if .IsNetworkTypeLiveMigration }}
+- apiGroups: ["network.openshift.io"]
+  resources:
+  - hostsubnets
+  verbs:
+  - create
+{{ end }}
 - apiGroups: [""]
   resources:
   - namespaces

diff --git a/bindata/network/openshift-sdn/controller.yaml b/bindata/network/openshift-sdn/controller.yaml
@@ -27,6 +27,11 @@ spec:
         - /bin/bash
         - -c
         - |
+{{- if .IsNetworkTypeLiveMigration }}
+          echo "wait for hostsubnets to be created by sdn pods"
+          # The hostsubnets needs be created according to the ovnkube node-subnets by the sdn pods before running the sdn controller.
+          while ! oc get nodes -o custom-columns=NAME:.metadata.name --no-headers| xargs -i oc get hostsubnet {}; do sleep 3; done
+{{- end }}
           if [[ -f /env/_master ]]; then
             set -o allexport
             source /env/_master

diff --git a/bindata/network/openshift-sdn/sdn.yaml b/bindata/network/openshift-sdn/sdn.yaml
@@ -66,7 +66,62 @@ spec:
         - |
           #!/bin/bash
           set -euo pipefail
+{{- if .IsNetworkTypeLiveMigration }}
+          # In case of rollback, openshift-sdn need to reuse the host subnet used by ovn-kube, so we need to create the hostsubnet CR for the local node if it doesn't exist.
+          # retry loop
+          RETRY_INTERVAL=3
+          MAX_RETRIES=10
+          for ((retry_count=1; retry_count<=MAX_RETRIES; retry_count++)); do
+              # Get SDN subnet
+              SDN_SUBNET=$(oc get hostsubnet ${K8S_NODE_NAME} -o jsonpath="{.subnet}"||true)
 
+              # Get OVN subnet
+              OVN_SUBNET=$(oc get node ${K8S_NODE_NAME} -o jsonpath='{.metadata.annotations.k8s\.ovn\.org/node-subnets}' | grep -o -E '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/[0-9]{1,3}'||true)
+
+              if [ -n "${SDN_SUBNET}" ] && [ -z "${OVN_SUBNET}" ]; then
+                  echo "SDN_SUBNET is ${SDN_SUBNET}"
+                  break
+              elif [ -z "${SDN_SUBNET}" ] && [ -n "${OVN_SUBNET}" ]; then
+                  echo "SDN_SUBNET is empty, create the hostsubnet CR with OVN_SUBNET ${OVN_SUBNET}"
+                  SUBNET=${OVN_SUBNET}
+                  NODE_UID=$(oc get node ${K8S_NODE_NAME} -o jsonpath='{.metadata.uid}')
+                  NODE_IP=$(oc get node ${K8S_NODE_NAME} -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}')
+          cat <<EOF | oc apply -f -
+          apiVersion: network.openshift.io/v1
+          kind: HostSubnet
+          metadata:
+            name: ${K8S_NODE_NAME}
+            annotations:
+              pod.network.openshift.io/node-uid: ${NODE_UID}
+          host: ${K8S_NODE_NAME}
+          hostIP: ${NODE_IP}
+          subnet: ${SUBNET}
+          EOF
+              elif [ -n "${SDN_SUBNET}" ] && [ -n "${OVN_SUBNET}" ] && [ "${SDN_SUBNET}" == "${OVN_SUBNET}" ]; then
+                  echo "Subnets matched! SDN_SUBNET: ${SDN_SUBNET}, OVN_SUBNET: ${OVN_SUBNET}"
+                  break
+              elif [ -n "${SDN_SUBNET}" ] && [ -n "${OVN_SUBNET}" ] && ! [ "${SDN_SUBNET}" == "${OVN_SUBNET}" ]; then
+                  echo "Subnets don't match! SDN_SUBNET: ${SDN_SUBNET}, OVN_SUBNET: ${OVN_SUBNET}"
+                  exit 1
+              else
+                  echo "Round ${retry_count}/${MAX_RETRIES} - Retrying... SDN_SUBNET: ${SDN_SUBNET}, OVN_SUBNET: ${OVN_SUBNET}"
+                  sleep $RETRY_INTERVAL
+              fi
+          done
+          if [ ${retry_count} -gt ${MAX_RETRIES} ]; then
+              echo "All retries failed. Exiting."
+              exit 1
+          fi
+          if ovs-vsctl br-exists br-ex; then
+            echo "br-ex exists, sleep..."
+            trap : TERM INT; sleep infinity & wait
+            exit
+          else
+            echo "br-ex doesn't exist"
+          fi
+          ovs-vsctl --if-exists del-br br-int
+          ovs-vsctl --if-exists del-br br-ext
+{{- end }}
           # if another process is listening on the cni-server socket, wait until it exits
           trap 'kill $(jobs -p); rm -f /etc/cni/net.d/80-openshift-network.conf ; exit 0' TERM
           retries=0
@@ -253,7 +308,18 @@ spec:
         readinessProbe:
           exec:
             # openshift-sdn writes this file when it is ready to handle pod requests.
-            command: ["test", "-f", "/etc/cni/net.d/80-openshift-network.conf"]
+            command: 
+            - /bin/bash
+            - -c
+            - |
+              #!/bin/bash
+              set -euo pipefail
+{{- if .IsNetworkTypeLiveMigration }}
+              if ovs-vsctl br-exists br-ex; then
+                exit 0
+              fi
+{{- end}}
+              test -f /etc/cni/net.d/80-openshift-network.conf
           initialDelaySeconds: 5
           periodSeconds: 5
       - name: kube-rbac-proxy

diff --git a/bindata/network/ovn-kubernetes/common/002-rbac-node.yaml b/bindata/network/ovn-kubernetes/common/002-rbac-node.yaml
@@ -186,7 +186,6 @@ rules:
   - get
   - list
   - watch
-
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -212,6 +211,41 @@ subjects:
   namespace: openshift-ovn-kubernetes
 {{ end }}
 
+{{ if .IsNetworkTypeLiveMigration }}
+# we need to kubectl command to patch nodes and pods before running ovnkube-node
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: openshift-ovn-kubernetes-network-type-live-migration
+rules:
+- apiGroups: [""]
+  resources:
+  - nodes
+  - pods
+  verbs:
+  - get
+  - patch
+- apiGroups: ['network.openshift.io']
+  resources:
+  - hostsubnets
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: openshift-ovn-kubernetes-network-type-live-migration
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: openshift-ovn-kubernetes-network-type-live-migration
+subjects:
+- kind: ServiceAccount
+  name: ovn-kubernetes-node
+  namespace: openshift-ovn-kubernetes
+{{ end }}
+
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole

diff --git a/bindata/network/ovn-kubernetes/common/008-script-lib.yaml b/bindata/network/ovn-kubernetes/common/008-script-lib.yaml
@@ -11,7 +11,15 @@ data:
   ovnkube-lib.sh: |-
     #!/bin/bash
     set -x
-
+{{- if .IsNetworkTypeLiveMigration }}
+    if ip link show br-ex; then
+      echo "br-ex exists"
+    else
+      echo "br-ex doesn't exist, sleep..."
+      trap : TERM INT; sleep infinity & wait
+      exit
+    fi
+{{- end }}
     # Add node-specific overrides if the container has mounted any
     K8S_NODE=${K8S_NODE:-}
     if [[ -n "${K8S_NODE}" && -f "/env/${K8S_NODE}" ]]; then

diff --git a/bindata/network/ovn-kubernetes/self-hosted/common/004-config.yaml b/bindata/network/ovn-kubernetes/self-hosted/common/004-config.yaml
@@ -26,6 +26,9 @@ data:
 {{- if .OVNHybridOverlayEnable }}
     no-hostsubnet-nodes="kubernetes.io/os=windows"
 {{- end  }}
+{{- if .IsNetworkTypeLiveMigration }}
+    no-hostsubnet-nodes="migration.network.openshift.io/plugin="
+{{- end }}
     platform-type="{{.PlatformType}}"
     healthz-bind-address="0.0.0.0:10256"
     dns-service-namespace="openshift-dns"
@@ -67,7 +70,7 @@ data:
     logfile-maxsize=100
     logfile-maxbackups=5
     logfile-maxage=0
-{{- if .OVNHybridOverlayEnable }}
+{{- if or .OVNHybridOverlayEnable .IsNetworkTypeLiveMigration}}
 
     [hybridoverlay]
     enabled=true

diff --git a/...ata/network/ovn-kubernetes/self-hosted/multi-zone-interconnect/ovnkube-control-plane.yaml b/...ata/network/ovn-kubernetes/self-hosted/multi-zone-interconnect/ovnkube-control-plane.yaml
@@ -95,6 +95,11 @@ spec:
         - -c
         - |
           set -xe
+{{- if .IsNetworkTypeLiveMigration }}
+          echo "wait for node-subnets node annotation in live migration mode"
+          # The node-subnets node annotation needs to be created according to the openshift-sdn hostsubnet CRs by ovnkube-node pods before running ovnkube-cluster-manager.
+          while kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.annotations.k8s\.ovn\.org/node-subnets}{"\n"}{end}'| grep -v -q "default"; do echo "subnet annotation is missing for some nodes" && sleep 3; done
+{{- end }}
           if [[ -f "/env/_master" ]]; then
             set -o allexport
             source "/env/_master"