diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml index a3bd42a6e76..345a0117350 100644 --- a/.github/workflows/build-x86-image.yaml +++ b/.github/workflows/build-x86-image.yaml @@ -2002,7 +2002,7 @@ jobs: - name: Create kind cluster env: - k8s_version: v1.23.17 + k8s_version: v1.29.10 run: | pipx install jinjanator make kind-init @@ -2421,6 +2421,186 @@ jobs: - name: Cleanup run: timeout -k 10 180 sh -x dist/images/cleanup.sh + vpc-egress-gateway-e2e: + name: VPC Egress Gateway E2E + needs: + - build-kube-ovn + - build-e2e-binaries + runs-on: ubuntu-24.04 + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + ip-family: + - ipv4 + - ipv6 + - dual + steps: + - uses: jlumbroso/free-disk-space@v1.3.1 + with: + android: true + dotnet: true + haskell: true + docker-images: false + large-packages: false + tool-cache: false + swap-storage: false + + - uses: actions/checkout@v4 + + - name: Create the default branch directory + if: (github.base_ref || github.ref_name) != github.event.repository.default_branch + run: mkdir -p test/e2e/source + + - name: Check out the default branch + if: (github.base_ref || github.ref_name) != github.event.repository.default_branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 1 + path: test/e2e/source + + - name: Export E2E directory + run: | + if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then + echo "E2E_DIR=." >> "$GITHUB_ENV" + else + echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV" + fi + + - uses: actions/setup-go@v5 + id: setup-go + with: + go-version-file: ${{ env.E2E_DIR }}/go.mod + check-latest: true + cache: false + + - name: Export Go full version + run: echo "GO_VERSION=${{ steps.setup-go.outputs.go-version }}" >> "$GITHUB_ENV" + + - name: Go cache + uses: actions/cache/restore@v4 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }} + restore-keys: ${{ runner.os }}-e2e-go-${{ env.GO_VERSION }}-x86- + + - name: Install kind + uses: helm/kind-action@v1.10.0 + with: + version: ${{ env.KIND_VERSION }} + install_only: true + + - name: Install ginkgo + working-directory: ${{ env.E2E_DIR }} + run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo + + - name: Download image + uses: actions/download-artifact@v4 + with: + name: kube-ovn + + - name: Load image + run: docker load --input kube-ovn.tar + + - name: Create kind cluster + run: | + pipx install jinjanator + make kind-init-${{ matrix.ip-family }} + + - name: Install Kube-OVN + id: install + run: make kind-install-debug-valgrind-${{ matrix.ip-family }} + + - name: Install Multus + run: make kind-install-multus + + - name: Run E2E + id: e2e + working-directory: ${{ env.E2E_DIR }} + env: + E2E_BRANCH: ${{ github.base_ref || github.ref_name }} + E2E_IP_FAMILY: ${{ matrix.ip-family }} + run: make vpc-egress-gateway-e2e + + - name: Collect k8s events + if: failure() && steps.e2e.conclusion == 'failure' + run: | + kubectl get events -A -o yaml > kube-ovn-conformance-e2e-${{ matrix.ip-family }}-events.yaml + tar zcf kube-ovn-conformance-e2e-${{ matrix.ip-family }}-events.tar.gz kube-ovn-conformance-e2e-${{ matrix.ip-family }}-events.yaml + + - name: Upload k8s events + uses: actions/upload-artifact@v4 + if: failure() && steps.e2e.conclusion == 'failure' + with: + name: kube-ovn-conformance-e2e-${{ matrix.ip-family }}-events + path: kube-ovn-conformance-e2e-${{ matrix.ip-family }}-events.tar.gz + + - name: Collect apiserver audit logs + if: failure() && steps.e2e.conclusion == 'failure' + run: | + docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log . + tar zcf kube-ovn-conformance-e2e-${{ matrix.ip-family }}-audit-log.tar.gz kube-apiserver-audit.log + + - name: Upload apiserver audit logs + uses: actions/upload-artifact@v4 + if: failure() && steps.e2e.conclusion == 'failure' + with: + name: kube-ovn-conformance-e2e-${{ matrix.ip-family }}-audit-log + path: kube-ovn-conformance-e2e-${{ matrix.ip-family }}-audit-log.tar.gz + + - name: kubectl ko log + if: failure() && steps.e2e.conclusion == 'failure' + run: | + make kubectl-ko-log + mv kubectl-ko-log.tar.gz kube-ovn-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz + + - name: upload kubectl ko log + uses: actions/upload-artifact@v4 + if: failure() && steps.e2e.conclusion == 'failure' + with: + name: kube-ovn-conformance-e2e-${{ matrix.ip-family }}-ko-log + path: kube-ovn-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz + + - name: Check kube ovn pod restarts + if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }} + run: make check-kube-ovn-pod-restarts + + - name: Check valgrind result + run: | + kubectl -n kube-system rollout restart ds ovs-ovn + kubectl -n kube-system rollout status ds ovs-ovn + sleep 10 + kubectl -n kube-system rollout restart deploy ovn-central + kubectl -n kube-system rollout status deploy ovn-central + while true; do + if [ $(kubectl -n kube-system get pod -l app=ovs -o name | wc -l) -eq $(kubectl get node -o name | wc -l) ]; then + break + fi + sleep 1 + done + kubectl ko log ovn + kubectl ko log ovs + + for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do + echo "Checking if valgrind log file for $daemon exists..." + find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1 + done + + find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do + if grep -qw 'definitely lost' "$f"; then + echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')." + echo $f + cat "$f" + exit 1 + fi; + done + + - name: Cleanup + run: timeout -k 10 180 sh -x dist/images/cleanup.sh + iptables-vpc-nat-gw-conformance-e2e: name: Iptables VPC NAT Gateway E2E needs: @@ -2966,6 +3146,7 @@ jobs: - kube-ovn-ic-conformance-e2e - kube-ovn-ipsec-e2e - multus-conformance-e2e + - vpc-egress-gateway-e2e - ovn-vpc-nat-gw-conformance-e2e - iptables-vpc-nat-gw-conformance-e2e - webhook-e2e diff --git a/Makefile.e2e b/Makefile.e2e index 39b1220fe42..b94734cebaa 100644 --- a/Makefile.e2e +++ b/Makefile.e2e @@ -74,6 +74,7 @@ e2e-build: ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/multus ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/lb-svc ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/vip + ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/vpc-egress-gateway ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/iptables-vpc-nat-gw ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ovn-vpc-nat-gw ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ha @@ -168,6 +169,14 @@ vip-conformance-e2e: ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \ --focus=CNI:Kube-OVN ./test/e2e/vip/vip.test -- $(TEST_BIN_ARGS) +.PHONY: vpc-egress-gateway-e2e +vpc-egress-gateway-e2e: + ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/vpc-egress-gateway + E2E_BRANCH=$(E2E_BRANCH) \ + E2E_IP_FAMILY=$(E2E_IP_FAMILY) \ + ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v --timeout=30m \ + --focus=CNI:Kube-OVN ./test/e2e/vpc-egress-gateway/vpc-egress-gateway.test -- $(TEST_BIN_ARGS) + .PHONY: iptables-vpc-nat-gw-conformance-e2e iptables-vpc-nat-gw-conformance-e2e: ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/iptables-vpc-nat-gw diff --git a/charts/kube-ovn/Chart.yaml b/charts/kube-ovn/Chart.yaml index 0978817b6fc..0db86bd5093 100644 --- a/charts/kube-ovn/Chart.yaml +++ b/charts/kube-ovn/Chart.yaml @@ -23,4 +23,4 @@ version: 1.13.0 # It is recommended to use it with quotes. appVersion: "1.13.0" -kubeVersion: ">= 1.23.0-0" +kubeVersion: ">= 1.29.0-0" diff --git a/charts/kube-ovn/templates/controller-deploy.yaml b/charts/kube-ovn/templates/controller-deploy.yaml index 06fdba72d95..b4364fa0adb 100644 --- a/charts/kube-ovn/templates/controller-deploy.yaml +++ b/charts/kube-ovn/templates/controller-deploy.yaml @@ -139,6 +139,7 @@ spec: - --enable-anp={{- .Values.func.ENABLE_ANP }} - --ovsdb-con-timeout={{- .Values.func.OVSDB_CON_TIMEOUT }} - --ovsdb-inactivity-timeout={{- .Values.func.OVSDB_INACTIVITY_TIMEOUT }} + - --image={{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} securityContext: runAsUser: {{ include "kubeovn.runAsUser" . }} privileged: false diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml index 1c858734813..f1be38730a7 100644 --- a/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -823,6 +823,275 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + name: vpc-egress-gateways.kubeovn.io +spec: + group: kubeovn.io + names: + plural: vpc-egress-gateways + singular: vpc-egress-gateway + shortNames: + - vpc-egress-gw + - veg + kind: VpcEgressGateway + listKind: VpcEgressGatewayList + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.vpc + name: VPC + type: string + - jsonPath: .spec.replicas + name: REPLICAS + type: integer + - jsonPath: .spec.bfd.enabled + name: BFD ENABLED + type: boolean + - jsonPath: .spec.externalSubnet + name: EXTERNAL SUBNET + type: string + - jsonPath: .status.phase + name: PHASE + type: string + - jsonPath: .status.ready + name: READY + type: boolean + - jsonPath: .status.internalIPs + name: INTERNAL IPS + priority: 1 + type: string + - jsonPath: .status.externalIPs + name: EXTERNAL IPS + priority: 1 + type: string + - jsonPath: .status.workload + name: WORKLOAD NAME + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + status: + properties: + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + lastUpdateTime: + format: date-time + type: string + message: + maxLength: 32768 + type: string + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + enum: + - "True" + - "False" + - Unknown + type: string + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - lastUpdateTime + - observedGeneration + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + internalIPs: + items: + type: string + type: array + externalIPs: + items: + type: string + type: array + phase: + type: string + default: Pending + enum: + - Pending + - Processing + - Completed + ready: + type: boolean + default: false + workload: + type: string + required: + - conditions + - phase + type: object + spec: + type: object + required: + - externalSubnet + x-kubernetes-validations: + - rule: "size(self.internalIPs) == 0 || size(self.internalIPs) >= self.replicas" + message: 'Size of Internal IPs MUST be equal to or greater than Replicas' + fieldPath: ".internalIPs" + - rule: "size(self.externalIPs) == 0 || size(self.externalIPs) >= self.replicas" + message: 'Size of External IPs MUST be equal to or greater than Replicas' + fieldPath: ".externalIPs" + properties: + replicas: + type: integer + default: 1 + minimum: 1 + maximum: 10 + vpc: + type: string + internalSubnet: + type: string + externalSubnet: + type: string + internalIPs: + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5]),((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ + - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:))),(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ + type: array + x-kubernetes-list-type: set + externalIPs: + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5]),((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ + - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:))),(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ + type: array + x-kubernetes-list-type: set + image: + type: string + bfd: + type: object + properties: + enabled: + type: boolean + default: false + minRX: + type: integer + default: 1000 + minTX: + type: integer + default: 1000 + multiplier: + type: integer + default: 3 + policies: + type: array + items: + type: object + properties: + snat: + type: boolean + default: false + ipBlocks: + type: array + x-kubernetes-list-type: set + items: + type: string + anyOf: + - format: ipv4 + - format: ipv6 + - format: cidr + subnets: + type: array + x-kubernetes-list-type: set + items: + type: string + minLength: 1 + nodeSelector: + type: array + items: + type: object + properties: + matchLabels: + additionalProperties: + type: string + type: object + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + type: array + x-kubernetes-list-type: set + items: + type: string + required: + - key + - operator + matchFields: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + type: array + x-kubernetes-list-type: set + items: + type: string + required: + - key + - operator +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: name: iptables-eips.kubeovn.io spec: @@ -1665,6 +1934,51 @@ spec: type: string type: object type: array + bfdPort: + properties: + enabled: + type: boolean + default: false + ip: + type: string + anyOf: + - pattern: ^$ + - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ + - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ + - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5]),((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ + - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:))),(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ + nodeSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + type: object + x-kubernetes-validations: + - rule: "self.enabled == false || self.ip != ''" + message: 'Port IP must be set when BFD Port is enabled' type: object status: properties: @@ -1721,6 +2035,20 @@ spec: type: string sctpSessionLoadBalancer: type: string + bfdPort: + type: object + properties: + enabled: + type: boolean + default: false + ip: + type: string + name: + type: string + nodes: + type: array + items: + type: string type: object type: object served: true diff --git a/charts/kube-ovn/templates/ovn-CR.yaml b/charts/kube-ovn/templates/ovn-CR.yaml index 856c9cd5b86..22fa5370854 100644 --- a/charts/kube-ovn/templates/ovn-CR.yaml +++ b/charts/kube-ovn/templates/ovn-CR.yaml @@ -13,6 +13,8 @@ rules: - vpcs/status - vpc-nat-gateways - vpc-nat-gateways/status + - vpc-egress-gateways + - vpc-egress-gateways/status - subnets - subnets/status - ippools @@ -98,6 +100,18 @@ rules: - daemonsets verbs: - get + - apiGroups: + - apps + resources: + - deployments + - deployments/scale + verbs: + - get + - list + - watch + - create + - update + - delete - apiGroups: - "" resources: @@ -124,8 +138,6 @@ rules: - apps resources: - statefulsets - - deployments - - deployments/scale verbs: - get - list diff --git a/charts/kube-ovn/templates/ovncni-ds.yaml b/charts/kube-ovn/templates/ovncni-ds.yaml index d0116317076..63f9dcf5224 100644 --- a/charts/kube-ovn/templates/ovncni-ds.yaml +++ b/charts/kube-ovn/templates/ovncni-ds.yaml @@ -35,11 +35,7 @@ spec: command: - sh - -xec - - {{ if not .Values.DISABLE_MODULES_MANAGEMENT -}} - iptables -V - {{- else -}} - echo "nothing to do" - {{- end }} + - iptables -V securityContext: allowPrivilegeEscalation: true capabilities: @@ -128,9 +124,6 @@ spec: - NET_RAW - SYS_ADMIN - SYS_PTRACE - {{- if not .Values.DISABLE_MODULES_MANAGEMENT }} - - SYS_MODULE - {{- end }} - SYS_NICE env: - name: ENABLE_SSL diff --git a/dist/images/Dockerfile.base b/dist/images/Dockerfile.base index a39863369ec..5c4020c62a7 100644 --- a/dist/images/Dockerfile.base +++ b/dist/images/Dockerfile.base @@ -61,7 +61,9 @@ RUN cd /usr/src/ && git clone -b branch-24.03 --depth=1 https://github.com/ovn-o # northd: skip arp/nd request for lrp addresses from localnet ports curl -s https://github.com/kubeovn/ovn/commit/283930b627ffa843ebf0e7c3fa0cc70edacfdd12.patch | git apply && \ # ovn-controller: make activation strategy work for single chassis - curl -s https://github.com/kubeovn/ovn/commit/1160d956e49e8f3f1b19535dbf1b9a624a090717.patch | git apply + curl -s https://github.com/kubeovn/ovn/commit/1160d956e49e8f3f1b19535dbf1b9a624a090717.patch | git apply && \ + # support dedicated BFD LRP + curl -s https://github.com/kubeovn/ovn/commit/40345aa35d03c93cde877ccfa8111346291ebc7c.patch | git apply RUN apt install -y build-essential fakeroot \ autoconf automake bzip2 debhelper-compat dh-exec dh-python dh-sequence-python3 dh-sequence-sphinxdoc \ @@ -89,10 +91,12 @@ RUN mkdir -p /usr/src/openbfdd && \ tar -xz -C /usr/src/openbfdd --strip-components=1 ADD OpenBFDD-compile.patch /usr/src/ +ADD OpenBFDD-allow-ttl-254.patch /usr/src/ RUN cd /usr/src/openbfdd && \ rm -vf missing && \ git apply --no-apply /usr/src/OpenBFDD-compile.patch && \ + git apply --no-apply /usr/src/OpenBFDD-allow-ttl-254.patch && \ autoupdate && \ ./autogen.sh && \ ./configure --enable-silent-rules && \ @@ -138,13 +142,14 @@ RUN apt update && apt upgrade -y && apt install ca-certificates python3 hostname setcap CAP_NET_RAW+eip $(readlink -f $(which ndisc6)) && \ setcap CAP_NET_RAW+eip $(readlink -f $(which tcpdump)) && \ setcap CAP_NET_ADMIN+eip $(readlink -f $(which ethtool)) && \ + setcap CAP_NET_ADMIN+eip $(readlink -f $(which conntrack)) && \ setcap CAP_SYS_ADMIN+eip $(readlink -f $(which nsenter)) && \ + setcap CAP_SYS_ADMIN+eip $(readlink -f $(which sysctl)) && \ setcap CAP_SYS_MODULE+eip $(readlink -f $(which modprobe)) && \ - setcap CAP_NET_ADMIN+eip $(readlink -f $(which conntrack)) && \ - setcap CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_MODULE+eip $(readlink -f $(which ipset)) && \ - setcap CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_MODULE+eip $(readlink -f $(which xtables-legacy-multi)) && \ - setcap CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_MODULE+eip $(readlink -f $(which xtables-nft-multi)) && \ - setcap CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_MODULE,CAP_SYS_ADMIN+eip $(readlink -f $(which ip)) && \ + setcap CAP_NET_RAW,CAP_NET_ADMIN+eip $(readlink -f $(which ipset)) && \ + setcap CAP_NET_RAW,CAP_NET_ADMIN+eip $(readlink -f $(which xtables-legacy-multi)) && \ + setcap CAP_NET_RAW,CAP_NET_ADMIN+eip $(readlink -f $(which xtables-nft-multi)) && \ + setcap CAP_NET_ADMIN,CAP_SYS_ADMIN+eip $(readlink -f $(which ip)) && \ rm -rf /var/lib/apt/lists/* && \ rm -rf /etc/localtime && \ rm -f /usr/bin/nc && \ diff --git a/dist/images/OpenBFDD-allow-ttl-254.patch b/dist/images/OpenBFDD-allow-ttl-254.patch new file mode 100644 index 00000000000..7172eae2895 --- /dev/null +++ b/dist/images/OpenBFDD-allow-ttl-254.patch @@ -0,0 +1,26 @@ +From edc60ecd05185acf317ac3ca67c54eb50f9e99a8 Mon Sep 17 00:00:00 2001 +From: zhangzujian +Date: Thu, 31 Oct 2024 02:56:09 +0000 +Subject: [PATCH] allow ttl 254 + +Signed-off-by: zhangzujian +--- + Beacon.cpp | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/Beacon.cpp b/Beacon.cpp +index d1b0658..1d4c3da 100755 +--- a/Beacon.cpp ++++ b/Beacon.cpp +@@ -481,7 +481,7 @@ void Beacon::handleListenSocket(Socket &socket) + } + + // TTL assumes that all control packets are from neighbors. +- if (ttl != 255) ++ if (ttl < 254) + { + gLog.Optional(Log::Discard, "Discard packet: bad ttl/hops %hhu", ttl); + return; +-- +2.43.0 + diff --git a/dist/images/bfdd-prestart.sh b/dist/images/bfdd-prestart.sh new file mode 100644 index 00000000000..eae5e68da7c --- /dev/null +++ b/dist/images/bfdd-prestart.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -ex + +bfdd-control session new set mintx "${BFD_MIN_TX:-1000}" +bfdd-control session new set minrx "${BFD_MIN_RX:-1000}" +bfdd-control session new set multi "${BFD_MULTI:-3}" + +PEER_IPS=($(echo "${BFD_PEER_IPS:-::}" | tr ',' ' ')) +for ip in ${PEER_IPS[*]}; do + bfdd-control allow ${ip} +done + +bfdd-control log type command no diff --git a/dist/images/cleanup.sh b/dist/images/cleanup.sh index 8c2d6c850e4..31e55a6e79f 100644 --- a/dist/images/cleanup.sh +++ b/dist/images/cleanup.sh @@ -125,6 +125,7 @@ kubectl delete --ignore-not-found crd \ security-groups.kubeovn.io \ ippools.kubeovn.io \ vpc-nat-gateways.kubeovn.io \ + vpc-egress-gateways.kubeovn.io \ vpcs.kubeovn.io \ vlans.kubeovn.io \ provider-networks.kubeovn.io \ diff --git a/dist/images/init-vpc-egress-gateway.sh b/dist/images/init-vpc-egress-gateway.sh new file mode 100644 index 00000000000..605d2421d0c --- /dev/null +++ b/dist/images/init-vpc-egress-gateway.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -ex + +INTERNAL_GATEWAY_IPV4=${INTERNAL_GATEWAY_IPV4:-} +INTERNAL_GATEWAY_IPV6=${INTERNAL_GATEWAY_IPV6:-} +INTERNAL_ROUTE_DST_IPV4=($(echo "${INTERNAL_ROUTE_DST_IPV4:-}" | tr ',' ' ')) +INTERNAL_ROUTE_DST_IPV6=($(echo "${INTERNAL_ROUTE_DST_IPV6:-}" | tr ',' ' ')) +SNAT_SOURCES_IPV4=($(echo "${SNAT_SOURCES_IPV4:-}" | tr ',' ' ')) +SNAT_SOURCES_IPV6=($(echo "${SNAT_SOURCES_IPV6:-}" | tr ',' ' ')) + +sysctl -w net.ipv4.ip_forward=1 +sysctl -w net.ipv6.conf.all.forwarding=1 + +iptables -V + +for dst in ${INTERNAL_ROUTE_DST_IPV4[*]}; do + ip route add "${dst}" via "${INTERNAL_GATEWAY_IPV4}" +done + +for dst in ${INTERNAL_ROUTE_DST_IPV6[*]}; do + ip route add "${dst}" via "${INTERNAL_GATEWAY_IPV6}" +done + +for src in ${SNAT_SOURCES_IPV4[*]}; do + iptables -t nat -A POSTROUTING -s "${src}" -j MASQUERADE --random-fully +done + +for src in ${SNAT_SOURCES_IPV6[*]}; do + ip6tables -t nat -A POSTROUTING -s "${src}" -j MASQUERADE --random-fully +done diff --git a/dist/images/install.sh b/dist/images/install.sh index b4683bb793b..d1596dad25d 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -1070,6 +1070,275 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + name: vpc-egress-gateways.kubeovn.io +spec: + group: kubeovn.io + names: + plural: vpc-egress-gateways + singular: vpc-egress-gateway + shortNames: + - vpc-egress-gw + - veg + kind: VpcEgressGateway + listKind: VpcEgressGatewayList + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.vpc + name: VPC + type: string + - jsonPath: .spec.replicas + name: REPLICAS + type: integer + - jsonPath: .spec.bfd.enabled + name: BFD ENABLED + type: boolean + - jsonPath: .spec.externalSubnet + name: EXTERNAL SUBNET + type: string + - jsonPath: .status.phase + name: PHASE + type: string + - jsonPath: .status.ready + name: READY + type: boolean + - jsonPath: .status.internalIPs + name: INTERNAL IPS + priority: 1 + type: string + - jsonPath: .status.externalIPs + name: EXTERNAL IPS + priority: 1 + type: string + - jsonPath: .status.workload + name: WORKLOAD NAME + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + status: + properties: + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + lastUpdateTime: + format: date-time + type: string + message: + maxLength: 32768 + type: string + observedGeneration: + format: int64 + minimum: 0 + type: integer + reason: + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + enum: + - "True" + - "False" + - Unknown + type: string + type: + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - lastUpdateTime + - observedGeneration + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + internalIPs: + items: + type: string + type: array + externalIPs: + items: + type: string + type: array + phase: + type: string + default: Pending + enum: + - Pending + - Processing + - Completed + ready: + type: boolean + default: false + workload: + type: string + required: + - conditions + - phase + type: object + spec: + type: object + required: + - externalSubnet + x-kubernetes-validations: + - rule: "size(self.internalIPs) == 0 || size(self.internalIPs) >= self.replicas" + message: 'Size of Internal IPs MUST be equal to or greater than Replicas' + fieldPath: ".internalIPs" + - rule: "size(self.externalIPs) == 0 || size(self.externalIPs) >= self.replicas" + message: 'Size of External IPs MUST be equal to or greater than Replicas' + fieldPath: ".externalIPs" + properties: + replicas: + type: integer + default: 1 + minimum: 1 + maximum: 10 + vpc: + type: string + internalSubnet: + type: string + externalSubnet: + type: string + internalIPs: + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5]),((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ + - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:))),(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ + type: array + x-kubernetes-list-type: set + externalIPs: + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5]),((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ + - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:))),(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ + type: array + x-kubernetes-list-type: set + image: + type: string + bfd: + type: object + properties: + enabled: + type: boolean + default: false + minRX: + type: integer + default: 1000 + minTX: + type: integer + default: 1000 + multiplier: + type: integer + default: 3 + policies: + type: array + items: + type: object + properties: + snat: + type: boolean + default: false + ipBlocks: + type: array + x-kubernetes-list-type: set + items: + type: string + anyOf: + - format: ipv4 + - format: ipv6 + - format: cidr + subnets: + type: array + x-kubernetes-list-type: set + items: + type: string + minLength: 1 + nodeSelector: + type: array + items: + type: object + properties: + matchLabels: + additionalProperties: + type: string + type: object + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + type: array + x-kubernetes-list-type: set + items: + type: string + required: + - key + - operator + matchFields: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + type: array + x-kubernetes-list-type: set + items: + type: string + required: + - key + - operator +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: name: iptables-eips.kubeovn.io spec: @@ -1912,6 +2181,51 @@ spec: type: string type: object type: array + bfdPort: + properties: + enabled: + type: boolean + default: false + ip: + type: string + anyOf: + - pattern: ^$ + - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ + - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ + - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5]),((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ + - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:))),(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ + nodeSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + type: object + x-kubernetes-validations: + - rule: "self.enabled == false || self.ip != ''" + message: 'Port IP must be set when BFD Port is enabled' type: object status: properties: @@ -1968,6 +2282,20 @@ spec: type: string sctpSessionLoadBalancer: type: string + bfdPort: + type: object + properties: + enabled: + type: boolean + default: false + ip: + type: string + name: + type: string + nodes: + type: array + items: + type: string type: object type: object served: true @@ -3017,6 +3345,8 @@ rules: - vpcs/status - vpc-nat-gateways - vpc-nat-gateways/status + - vpc-egress-gateways + - vpc-egress-gateways/status - subnets - subnets/status - ippools @@ -3102,6 +3432,18 @@ rules: - daemonsets verbs: - get + - apiGroups: + - apps + resources: + - deployments + - deployments/scale + verbs: + - get + - list + - watch + - create + - update + - delete - apiGroups: - "" resources: @@ -3128,8 +3470,6 @@ rules: - apps resources: - statefulsets - - deployments - - deployments/scale verbs: - get - list @@ -4328,6 +4668,7 @@ spec: - --enable-anp=$ENABLE_ANP - --ovsdb-con-timeout=$OVSDB_CON_TIMEOUT - --ovsdb-inactivity-timeout=$OVSDB_INACTIVITY_TIMEOUT + - --image=$REGISTRY/kube-ovn:$VERSION securityContext: runAsUser: ${RUN_AS_USER} privileged: false @@ -4526,7 +4867,6 @@ spec: - NET_BIND_SERVICE - NET_RAW - SYS_ADMIN - - SYS_MODULE - SYS_NICE - SYS_PTRACE env: diff --git a/dist/images/kubectl-ko b/dist/images/kubectl-ko index d0666d40c47..bef1de8a5c8 100755 --- a/dist/images/kubectl-ko +++ b/dist/images/kubectl-ko @@ -420,11 +420,11 @@ trace(){ case $type in icmp) set -x - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace "$ls" "inport == \"$lsp\" && ip.ttl == 64 && icmp && eth.src == $mac && ip$af.src == $srcIP && eth.dst == $dstMac && ip$af.dst == $dst && ct.new" + kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace "$ls" "inport == \"$lsp\" && ip.ttl == 255 && icmp && eth.src == $mac && ip$af.src == $srcIP && eth.dst == $dstMac && ip$af.dst == $dst && ct.new" ;; tcp|udp) set -x - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace "$ls" "inport == \"$lsp\" && ip.ttl == 64 && eth.src == $mac && ip$af.src == $srcIP && eth.dst == $dstMac && ip$af.dst == $dst && $type.src == 10000 && $type.dst == $4 && ct.new" + kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace "$ls" "inport == \"$lsp\" && ip.ttl == 255 && eth.src == $mac && ip$af.src == $srcIP && eth.dst == $dstMac && ip$af.dst == $dst && $type.src == 10000 && $type.dst == $4 && ct.new" ;; arp) case "$4" in @@ -625,6 +625,7 @@ diagnose(){ gcCommands=() kubectl get crd vpcs.kubeovn.io kubectl get crd vpc-nat-gateways.kubeovn.io + kubectl get crd vpc-egress-gateways.kubeovn.io kubectl get crd subnets.kubeovn.io kubectl get crd ips.kubeovn.io kubectl get crd vlans.kubeovn.io diff --git a/dist/images/start-bfdd.sh b/dist/images/start-bfdd.sh new file mode 100644 index 00000000000..f186ebbe8bd --- /dev/null +++ b/dist/images/start-bfdd.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -ex + +POD_IPS=${POD_IPS:-::} +LISTEN_IPS=($(echo "${POD_IPS}" | tr ',' ' ')) +LISTEN_ARGS="" +for ip in ${LISTEN_IPS[*]}; do + LISTEN_ARGS="${LISTEN_ARGS} --listen=${ip}" +done + +bfdd-beacon --nofork --tee ${LISTEN_ARGS} diff --git a/mocks/pkg/ovs/interface.go b/mocks/pkg/ovs/interface.go index e9c472beeb2..98cbc0553c9 100644 --- a/mocks/pkg/ovs/interface.go +++ b/mocks/pkg/ovs/interface.go @@ -469,6 +469,34 @@ func (mr *MockLogicalRouterPortMockRecorder) LogicalRouterPortExists(lrpName any return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LogicalRouterPortExists", reflect.TypeOf((*MockLogicalRouterPort)(nil).LogicalRouterPortExists), lrpName) } +// SetLogicalRouterPortHAChassisGroup mocks base method. +func (m *MockLogicalRouterPort) SetLogicalRouterPortHAChassisGroup(lrpName, haChassisGroupName string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetLogicalRouterPortHAChassisGroup", lrpName, haChassisGroupName) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetLogicalRouterPortHAChassisGroup indicates an expected call of SetLogicalRouterPortHAChassisGroup. +func (mr *MockLogicalRouterPortMockRecorder) SetLogicalRouterPortHAChassisGroup(lrpName, haChassisGroupName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLogicalRouterPortHAChassisGroup", reflect.TypeOf((*MockLogicalRouterPort)(nil).SetLogicalRouterPortHAChassisGroup), lrpName, haChassisGroupName) +} + +// UpdateLogicalRouterPortNetworks mocks base method. +func (m *MockLogicalRouterPort) UpdateLogicalRouterPortNetworks(lrpName string, networks []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateLogicalRouterPortNetworks", lrpName, networks) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateLogicalRouterPortNetworks indicates an expected call of UpdateLogicalRouterPortNetworks. +func (mr *MockLogicalRouterPortMockRecorder) UpdateLogicalRouterPortNetworks(lrpName, networks any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateLogicalRouterPortNetworks", reflect.TypeOf((*MockLogicalRouterPort)(nil).UpdateLogicalRouterPortNetworks), lrpName, networks) +} + // UpdateLogicalRouterPortOptions mocks base method. func (m *MockLogicalRouterPort) UpdateLogicalRouterPortOptions(lrpName string, options map[string]string) error { m.ctrl.T.Helper() @@ -497,6 +525,73 @@ func (mr *MockLogicalRouterPortMockRecorder) UpdateLogicalRouterPortRA(lrpName, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateLogicalRouterPortRA", reflect.TypeOf((*MockLogicalRouterPort)(nil).UpdateLogicalRouterPortRA), lrpName, ipv6RAConfigsStr, enableIPv6RA) } +// MockHAChassisGroup is a mock of HAChassisGroup interface. +type MockHAChassisGroup struct { + ctrl *gomock.Controller + recorder *MockHAChassisGroupMockRecorder + isgomock struct{} +} + +// MockHAChassisGroupMockRecorder is the mock recorder for MockHAChassisGroup. +type MockHAChassisGroupMockRecorder struct { + mock *MockHAChassisGroup +} + +// NewMockHAChassisGroup creates a new mock instance. +func NewMockHAChassisGroup(ctrl *gomock.Controller) *MockHAChassisGroup { + mock := &MockHAChassisGroup{ctrl: ctrl} + mock.recorder = &MockHAChassisGroupMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockHAChassisGroup) EXPECT() *MockHAChassisGroupMockRecorder { + return m.recorder +} + +// CreateHAChassisGroup mocks base method. +func (m *MockHAChassisGroup) CreateHAChassisGroup(name string, chassises []string, externalIDs map[string]string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateHAChassisGroup", name, chassises, externalIDs) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateHAChassisGroup indicates an expected call of CreateHAChassisGroup. +func (mr *MockHAChassisGroupMockRecorder) CreateHAChassisGroup(name, chassises, externalIDs any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateHAChassisGroup", reflect.TypeOf((*MockHAChassisGroup)(nil).CreateHAChassisGroup), name, chassises, externalIDs) +} + +// DeleteHAChassisGroup mocks base method. +func (m *MockHAChassisGroup) DeleteHAChassisGroup(name string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteHAChassisGroup", name) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteHAChassisGroup indicates an expected call of DeleteHAChassisGroup. +func (mr *MockHAChassisGroupMockRecorder) DeleteHAChassisGroup(name any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteHAChassisGroup", reflect.TypeOf((*MockHAChassisGroup)(nil).DeleteHAChassisGroup), name) +} + +// GetHAChassisGroup mocks base method. +func (m *MockHAChassisGroup) GetHAChassisGroup(name string, ignoreNotFound bool) (*ovnnb.HAChassisGroup, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetHAChassisGroup", name, ignoreNotFound) + ret0, _ := ret[0].(*ovnnb.HAChassisGroup) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetHAChassisGroup indicates an expected call of GetHAChassisGroup. +func (mr *MockHAChassisGroupMockRecorder) GetHAChassisGroup(name, ignoreNotFound any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHAChassisGroup", reflect.TypeOf((*MockHAChassisGroup)(nil).GetHAChassisGroup), name, ignoreNotFound) +} + // MockGatewayChassis is a mock of GatewayChassis interface. type MockGatewayChassis struct { ctrl *gomock.Controller @@ -565,32 +660,61 @@ func (m *MockBFD) EXPECT() *MockBFDMockRecorder { } // CreateBFD mocks base method. -func (m *MockBFD) CreateBFD(lrpName, dstIP string, minRx, minTx, detectMult int) (*ovnnb.BFD, error) { +func (m *MockBFD) CreateBFD(lrpName, dstIP string, minRx, minTx, detectMult int, externalIDs map[string]string) (*ovnnb.BFD, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateBFD", lrpName, dstIP, minRx, minTx, detectMult) + ret := m.ctrl.Call(m, "CreateBFD", lrpName, dstIP, minRx, minTx, detectMult, externalIDs) ret0, _ := ret[0].(*ovnnb.BFD) ret1, _ := ret[1].(error) return ret0, ret1 } // CreateBFD indicates an expected call of CreateBFD. -func (mr *MockBFDMockRecorder) CreateBFD(lrpName, dstIP, minRx, minTx, detectMult any) *gomock.Call { +func (mr *MockBFDMockRecorder) CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, externalIDs any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateBFD", reflect.TypeOf((*MockBFD)(nil).CreateBFD), lrpName, dstIP, minRx, minTx, detectMult) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateBFD", reflect.TypeOf((*MockBFD)(nil).CreateBFD), lrpName, dstIP, minRx, minTx, detectMult, externalIDs) } // DeleteBFD mocks base method. -func (m *MockBFD) DeleteBFD(lrpName, dstIP string) error { +func (m *MockBFD) DeleteBFD(uuid string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteBFD", lrpName, dstIP) + ret := m.ctrl.Call(m, "DeleteBFD", uuid) ret0, _ := ret[0].(error) return ret0 } // DeleteBFD indicates an expected call of DeleteBFD. -func (mr *MockBFDMockRecorder) DeleteBFD(lrpName, dstIP any) *gomock.Call { +func (mr *MockBFDMockRecorder) DeleteBFD(uuid any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBFD", reflect.TypeOf((*MockBFD)(nil).DeleteBFD), uuid) +} + +// DeleteBFDByDstIP mocks base method. +func (m *MockBFD) DeleteBFDByDstIP(lrpName, dstIP string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteBFDByDstIP", lrpName, dstIP) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteBFDByDstIP indicates an expected call of DeleteBFDByDstIP. +func (mr *MockBFDMockRecorder) DeleteBFDByDstIP(lrpName, dstIP any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBFDByDstIP", reflect.TypeOf((*MockBFD)(nil).DeleteBFDByDstIP), lrpName, dstIP) +} + +// FindBFD mocks base method. +func (m *MockBFD) FindBFD(externalIDs map[string]string) ([]ovnnb.BFD, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FindBFD", externalIDs) + ret0, _ := ret[0].([]ovnnb.BFD) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FindBFD indicates an expected call of FindBFD. +func (mr *MockBFDMockRecorder) FindBFD(externalIDs any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBFD", reflect.TypeOf((*MockBFD)(nil).DeleteBFD), lrpName, dstIP) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindBFD", reflect.TypeOf((*MockBFD)(nil).FindBFD), externalIDs) } // ListBFDs mocks base method. @@ -2066,6 +2190,34 @@ func (mr *MockLogicalRouterStaticRouteMockRecorder) DeleteLogicalRouterStaticRou return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLogicalRouterStaticRoute", reflect.TypeOf((*MockLogicalRouterStaticRoute)(nil).DeleteLogicalRouterStaticRoute), lrName, routeTable, policy, ipPrefix, nextHop) } +// DeleteLogicalRouterStaticRouteByExternalIDs mocks base method. +func (m *MockLogicalRouterStaticRoute) DeleteLogicalRouterStaticRouteByExternalIDs(lrName string, externalIDs map[string]string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteLogicalRouterStaticRouteByExternalIDs", lrName, externalIDs) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteLogicalRouterStaticRouteByExternalIDs indicates an expected call of DeleteLogicalRouterStaticRouteByExternalIDs. +func (mr *MockLogicalRouterStaticRouteMockRecorder) DeleteLogicalRouterStaticRouteByExternalIDs(lrName, externalIDs any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLogicalRouterStaticRouteByExternalIDs", reflect.TypeOf((*MockLogicalRouterStaticRoute)(nil).DeleteLogicalRouterStaticRouteByExternalIDs), lrName, externalIDs) +} + +// DeleteLogicalRouterStaticRouteByUUID mocks base method. +func (m *MockLogicalRouterStaticRoute) DeleteLogicalRouterStaticRouteByUUID(lrName, uuid string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteLogicalRouterStaticRouteByUUID", lrName, uuid) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteLogicalRouterStaticRouteByUUID indicates an expected call of DeleteLogicalRouterStaticRouteByUUID. +func (mr *MockLogicalRouterStaticRouteMockRecorder) DeleteLogicalRouterStaticRouteByUUID(lrName, uuid any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLogicalRouterStaticRouteByUUID", reflect.TypeOf((*MockLogicalRouterStaticRoute)(nil).DeleteLogicalRouterStaticRouteByUUID), lrName, uuid) +} + // ListLogicalRouterStaticRoutes mocks base method. func (m *MockLogicalRouterStaticRoute) ListLogicalRouterStaticRoutes(lrName string, routeTable, policy *string, ipPrefix string, externalIDs map[string]string) ([]*ovnnb.LogicalRouterStaticRoute, error) { m.ctrl.T.Helper() @@ -2111,6 +2263,25 @@ func (mr *MockLogicalRouterStaticRouteMockRecorder) LogicalRouterStaticRouteExis return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LogicalRouterStaticRouteExists", reflect.TypeOf((*MockLogicalRouterStaticRoute)(nil).LogicalRouterStaticRouteExists), lrName, routeTable, policy, ipPrefix, nexthop) } +// UpdateLogicalRouterStaticRoute mocks base method. +func (m *MockLogicalRouterStaticRoute) UpdateLogicalRouterStaticRoute(route *ovnnb.LogicalRouterStaticRoute, fields ...any) error { + m.ctrl.T.Helper() + varargs := []any{route} + for _, a := range fields { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "UpdateLogicalRouterStaticRoute", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateLogicalRouterStaticRoute indicates an expected call of UpdateLogicalRouterStaticRoute. +func (mr *MockLogicalRouterStaticRouteMockRecorder) UpdateLogicalRouterStaticRoute(route any, fields ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{route}, fields...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateLogicalRouterStaticRoute", reflect.TypeOf((*MockLogicalRouterStaticRoute)(nil).UpdateLogicalRouterStaticRoute), varargs...) +} + // MockLogicalRouterPolicy is a mock of LogicalRouterPolicy interface. type MockLogicalRouterPolicy struct { ctrl *gomock.Controller @@ -2136,17 +2307,17 @@ func (m *MockLogicalRouterPolicy) EXPECT() *MockLogicalRouterPolicyMockRecorder } // AddLogicalRouterPolicy mocks base method. -func (m *MockLogicalRouterPolicy) AddLogicalRouterPolicy(lrName string, priority int, match, action string, nextHops []string, externalIDs map[string]string) error { +func (m *MockLogicalRouterPolicy) AddLogicalRouterPolicy(lrName string, priority int, match, action string, nextHops, bfdSessions []string, externalIDs map[string]string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddLogicalRouterPolicy", lrName, priority, match, action, nextHops, externalIDs) + ret := m.ctrl.Call(m, "AddLogicalRouterPolicy", lrName, priority, match, action, nextHops, bfdSessions, externalIDs) ret0, _ := ret[0].(error) return ret0 } // AddLogicalRouterPolicy indicates an expected call of AddLogicalRouterPolicy. -func (mr *MockLogicalRouterPolicyMockRecorder) AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, externalIDs any) *gomock.Call { +func (mr *MockLogicalRouterPolicyMockRecorder) AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, bfdSessions, externalIDs any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddLogicalRouterPolicy", reflect.TypeOf((*MockLogicalRouterPolicy)(nil).AddLogicalRouterPolicy), lrName, priority, match, action, nextHops, externalIDs) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddLogicalRouterPolicy", reflect.TypeOf((*MockLogicalRouterPolicy)(nil).AddLogicalRouterPolicy), lrName, priority, match, action, nextHops, bfdSessions, externalIDs) } // ClearLogicalRouterPolicy mocks base method. @@ -2264,6 +2435,25 @@ func (mr *MockLogicalRouterPolicyMockRecorder) ListLogicalRouterPolicies(lrName, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListLogicalRouterPolicies", reflect.TypeOf((*MockLogicalRouterPolicy)(nil).ListLogicalRouterPolicies), lrName, priority, externalIDs, ignoreExtIDEmptyValue) } +// UpdateLogicalRouterPolicy mocks base method. +func (m *MockLogicalRouterPolicy) UpdateLogicalRouterPolicy(policy *ovnnb.LogicalRouterPolicy, fields ...any) error { + m.ctrl.T.Helper() + varargs := []any{policy} + for _, a := range fields { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "UpdateLogicalRouterPolicy", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateLogicalRouterPolicy indicates an expected call of UpdateLogicalRouterPolicy. +func (mr *MockLogicalRouterPolicyMockRecorder) UpdateLogicalRouterPolicy(policy any, fields ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{policy}, fields...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateLogicalRouterPolicy", reflect.TypeOf((*MockLogicalRouterPolicy)(nil).UpdateLogicalRouterPolicy), varargs...) +} + // MockNAT is a mock of NAT interface. type MockNAT struct { ctrl *gomock.Controller @@ -2528,17 +2718,17 @@ func (mr *MockNbClientMockRecorder) AddLoadBalancerHealthCheck(lbName, vip, exte } // AddLogicalRouterPolicy mocks base method. -func (m *MockNbClient) AddLogicalRouterPolicy(lrName string, priority int, match, action string, nextHops []string, externalIDs map[string]string) error { +func (m *MockNbClient) AddLogicalRouterPolicy(lrName string, priority int, match, action string, nextHops, bfdSessions []string, externalIDs map[string]string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddLogicalRouterPolicy", lrName, priority, match, action, nextHops, externalIDs) + ret := m.ctrl.Call(m, "AddLogicalRouterPolicy", lrName, priority, match, action, nextHops, bfdSessions, externalIDs) ret0, _ := ret[0].(error) return ret0 } // AddLogicalRouterPolicy indicates an expected call of AddLogicalRouterPolicy. -func (mr *MockNbClientMockRecorder) AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, externalIDs any) *gomock.Call { +func (mr *MockNbClientMockRecorder) AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, bfdSessions, externalIDs any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddLogicalRouterPolicy", reflect.TypeOf((*MockNbClient)(nil).AddLogicalRouterPolicy), lrName, priority, match, action, nextHops, externalIDs) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddLogicalRouterPolicy", reflect.TypeOf((*MockNbClient)(nil).AddLogicalRouterPolicy), lrName, priority, match, action, nextHops, bfdSessions, externalIDs) } // AddLogicalRouterStaticRoute mocks base method. @@ -2650,18 +2840,18 @@ func (mr *MockNbClientMockRecorder) CreateAddressSet(asName, externalIDs any) *g } // CreateBFD mocks base method. -func (m *MockNbClient) CreateBFD(lrpName, dstIP string, minRx, minTx, detectMult int) (*ovnnb.BFD, error) { +func (m *MockNbClient) CreateBFD(lrpName, dstIP string, minRx, minTx, detectMult int, externalIDs map[string]string) (*ovnnb.BFD, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateBFD", lrpName, dstIP, minRx, minTx, detectMult) + ret := m.ctrl.Call(m, "CreateBFD", lrpName, dstIP, minRx, minTx, detectMult, externalIDs) ret0, _ := ret[0].(*ovnnb.BFD) ret1, _ := ret[1].(error) return ret0, ret1 } // CreateBFD indicates an expected call of CreateBFD. -func (mr *MockNbClientMockRecorder) CreateBFD(lrpName, dstIP, minRx, minTx, detectMult any) *gomock.Call { +func (mr *MockNbClientMockRecorder) CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, externalIDs any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateBFD", reflect.TypeOf((*MockNbClient)(nil).CreateBFD), lrpName, dstIP, minRx, minTx, detectMult) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateBFD", reflect.TypeOf((*MockNbClient)(nil).CreateBFD), lrpName, dstIP, minRx, minTx, detectMult, externalIDs) } // CreateBareLogicalSwitch mocks base method. @@ -2725,6 +2915,20 @@ func (mr *MockNbClientMockRecorder) CreateGatewayLogicalSwitch(lsName, lrName, p return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateGatewayLogicalSwitch", reflect.TypeOf((*MockNbClient)(nil).CreateGatewayLogicalSwitch), varargs...) } +// CreateHAChassisGroup mocks base method. +func (m *MockNbClient) CreateHAChassisGroup(name string, chassises []string, externalIDs map[string]string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateHAChassisGroup", name, chassises, externalIDs) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateHAChassisGroup indicates an expected call of CreateHAChassisGroup. +func (mr *MockNbClientMockRecorder) CreateHAChassisGroup(name, chassises, externalIDs any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateHAChassisGroup", reflect.TypeOf((*MockNbClient)(nil).CreateHAChassisGroup), name, chassises, externalIDs) +} + // CreateLoadBalancer mocks base method. func (m *MockNbClient) CreateLoadBalancer(lbName, protocol, selectFields string) error { m.ctrl.T.Helper() @@ -3007,17 +3211,31 @@ func (mr *MockNbClientMockRecorder) DeleteAddressSets(externalIDs any) *gomock.C } // DeleteBFD mocks base method. -func (m *MockNbClient) DeleteBFD(lrpName, dstIP string) error { +func (m *MockNbClient) DeleteBFD(uuid string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteBFD", lrpName, dstIP) + ret := m.ctrl.Call(m, "DeleteBFD", uuid) ret0, _ := ret[0].(error) return ret0 } // DeleteBFD indicates an expected call of DeleteBFD. -func (mr *MockNbClientMockRecorder) DeleteBFD(lrpName, dstIP any) *gomock.Call { +func (mr *MockNbClientMockRecorder) DeleteBFD(uuid any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBFD", reflect.TypeOf((*MockNbClient)(nil).DeleteBFD), lrpName, dstIP) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBFD", reflect.TypeOf((*MockNbClient)(nil).DeleteBFD), uuid) +} + +// DeleteBFDByDstIP mocks base method. +func (m *MockNbClient) DeleteBFDByDstIP(lrpName, dstIP string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteBFDByDstIP", lrpName, dstIP) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteBFDByDstIP indicates an expected call of DeleteBFDByDstIP. +func (mr *MockNbClientMockRecorder) DeleteBFDByDstIP(lrpName, dstIP any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBFDByDstIP", reflect.TypeOf((*MockNbClient)(nil).DeleteBFDByDstIP), lrpName, dstIP) } // DeleteDHCPOptions mocks base method. @@ -3052,6 +3270,20 @@ func (mr *MockNbClientMockRecorder) DeleteDHCPOptionsByUUIDs(uuidList ...any) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteDHCPOptionsByUUIDs", reflect.TypeOf((*MockNbClient)(nil).DeleteDHCPOptionsByUUIDs), uuidList...) } +// DeleteHAChassisGroup mocks base method. +func (m *MockNbClient) DeleteHAChassisGroup(name string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteHAChassisGroup", name) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteHAChassisGroup indicates an expected call of DeleteHAChassisGroup. +func (mr *MockNbClientMockRecorder) DeleteHAChassisGroup(name any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteHAChassisGroup", reflect.TypeOf((*MockNbClient)(nil).DeleteHAChassisGroup), name) +} + // DeleteLoadBalancerHealthCheck mocks base method. func (m *MockNbClient) DeleteLoadBalancerHealthCheck(lbName, vip string) error { m.ctrl.T.Helper() @@ -3220,6 +3452,34 @@ func (mr *MockNbClientMockRecorder) DeleteLogicalRouterStaticRoute(lrName, route return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLogicalRouterStaticRoute", reflect.TypeOf((*MockNbClient)(nil).DeleteLogicalRouterStaticRoute), lrName, routeTable, policy, ipPrefix, nextHop) } +// DeleteLogicalRouterStaticRouteByExternalIDs mocks base method. +func (m *MockNbClient) DeleteLogicalRouterStaticRouteByExternalIDs(lrName string, externalIDs map[string]string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteLogicalRouterStaticRouteByExternalIDs", lrName, externalIDs) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteLogicalRouterStaticRouteByExternalIDs indicates an expected call of DeleteLogicalRouterStaticRouteByExternalIDs. +func (mr *MockNbClientMockRecorder) DeleteLogicalRouterStaticRouteByExternalIDs(lrName, externalIDs any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLogicalRouterStaticRouteByExternalIDs", reflect.TypeOf((*MockNbClient)(nil).DeleteLogicalRouterStaticRouteByExternalIDs), lrName, externalIDs) +} + +// DeleteLogicalRouterStaticRouteByUUID mocks base method. +func (m *MockNbClient) DeleteLogicalRouterStaticRouteByUUID(lrName, uuid string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteLogicalRouterStaticRouteByUUID", lrName, uuid) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteLogicalRouterStaticRouteByUUID indicates an expected call of DeleteLogicalRouterStaticRouteByUUID. +func (mr *MockNbClientMockRecorder) DeleteLogicalRouterStaticRouteByUUID(lrName, uuid any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLogicalRouterStaticRouteByUUID", reflect.TypeOf((*MockNbClient)(nil).DeleteLogicalRouterStaticRouteByUUID), lrName, uuid) +} + // DeleteLogicalSwitch mocks base method. func (m *MockNbClient) DeleteLogicalSwitch(lsName string) error { m.ctrl.T.Helper() @@ -3336,6 +3596,21 @@ func (mr *MockNbClientMockRecorder) EnablePortLayer2forward(lspName any) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnablePortLayer2forward", reflect.TypeOf((*MockNbClient)(nil).EnablePortLayer2forward), lspName) } +// FindBFD mocks base method. +func (m *MockNbClient) FindBFD(externalIDs map[string]string) ([]ovnnb.BFD, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FindBFD", externalIDs) + ret0, _ := ret[0].([]ovnnb.BFD) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FindBFD indicates an expected call of FindBFD. +func (mr *MockNbClientMockRecorder) FindBFD(externalIDs any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindBFD", reflect.TypeOf((*MockNbClient)(nil).FindBFD), externalIDs) +} + // GetEntityInfo mocks base method. func (m *MockNbClient) GetEntityInfo(entity any) error { m.ctrl.T.Helper() @@ -3350,6 +3625,21 @@ func (mr *MockNbClientMockRecorder) GetEntityInfo(entity any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetEntityInfo", reflect.TypeOf((*MockNbClient)(nil).GetEntityInfo), entity) } +// GetHAChassisGroup mocks base method. +func (m *MockNbClient) GetHAChassisGroup(name string, ignoreNotFound bool) (*ovnnb.HAChassisGroup, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetHAChassisGroup", name, ignoreNotFound) + ret0, _ := ret[0].(*ovnnb.HAChassisGroup) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetHAChassisGroup indicates an expected call of GetHAChassisGroup. +func (mr *MockNbClientMockRecorder) GetHAChassisGroup(name, ignoreNotFound any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHAChassisGroup", reflect.TypeOf((*MockNbClient)(nil).GetHAChassisGroup), name, ignoreNotFound) +} + // GetLoadBalancer mocks base method. func (m *MockNbClient) GetLoadBalancer(lbName string, ignoreNotFound bool) (*ovnnb.LoadBalancer, error) { m.ctrl.T.Helper() @@ -4254,6 +4544,20 @@ func (mr *MockNbClientMockRecorder) SetLoadBalancerAffinityTimeout(lbName, timeo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLoadBalancerAffinityTimeout", reflect.TypeOf((*MockNbClient)(nil).SetLoadBalancerAffinityTimeout), lbName, timeout) } +// SetLogicalRouterPortHAChassisGroup mocks base method. +func (m *MockNbClient) SetLogicalRouterPortHAChassisGroup(lrpName, haChassisGroupName string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetLogicalRouterPortHAChassisGroup", lrpName, haChassisGroupName) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetLogicalRouterPortHAChassisGroup indicates an expected call of SetLogicalRouterPortHAChassisGroup. +func (mr *MockNbClientMockRecorder) SetLogicalRouterPortHAChassisGroup(lrpName, haChassisGroupName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLogicalRouterPortHAChassisGroup", reflect.TypeOf((*MockNbClient)(nil).SetLogicalRouterPortHAChassisGroup), lrpName, haChassisGroupName) +} + // SetLogicalSwitchPortActivationStrategy mocks base method. func (m *MockNbClient) SetLogicalSwitchPortActivationStrategy(lspName, chassis string) error { m.ctrl.T.Helper() @@ -4614,6 +4918,39 @@ func (mr *MockNbClientMockRecorder) UpdateLogicalRouter(lr any, fields ...any) * return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateLogicalRouter", reflect.TypeOf((*MockNbClient)(nil).UpdateLogicalRouter), varargs...) } +// UpdateLogicalRouterPolicy mocks base method. +func (m *MockNbClient) UpdateLogicalRouterPolicy(policy *ovnnb.LogicalRouterPolicy, fields ...any) error { + m.ctrl.T.Helper() + varargs := []any{policy} + for _, a := range fields { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "UpdateLogicalRouterPolicy", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateLogicalRouterPolicy indicates an expected call of UpdateLogicalRouterPolicy. +func (mr *MockNbClientMockRecorder) UpdateLogicalRouterPolicy(policy any, fields ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{policy}, fields...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateLogicalRouterPolicy", reflect.TypeOf((*MockNbClient)(nil).UpdateLogicalRouterPolicy), varargs...) +} + +// UpdateLogicalRouterPortNetworks mocks base method. +func (m *MockNbClient) UpdateLogicalRouterPortNetworks(lrpName string, networks []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateLogicalRouterPortNetworks", lrpName, networks) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateLogicalRouterPortNetworks indicates an expected call of UpdateLogicalRouterPortNetworks. +func (mr *MockNbClientMockRecorder) UpdateLogicalRouterPortNetworks(lrpName, networks any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateLogicalRouterPortNetworks", reflect.TypeOf((*MockNbClient)(nil).UpdateLogicalRouterPortNetworks), lrpName, networks) +} + // UpdateLogicalRouterPortOptions mocks base method. func (m *MockNbClient) UpdateLogicalRouterPortOptions(lrpName string, options map[string]string) error { m.ctrl.T.Helper() @@ -4642,6 +4979,25 @@ func (mr *MockNbClientMockRecorder) UpdateLogicalRouterPortRA(lrpName, ipv6RACon return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateLogicalRouterPortRA", reflect.TypeOf((*MockNbClient)(nil).UpdateLogicalRouterPortRA), lrpName, ipv6RAConfigsStr, enableIPv6RA) } +// UpdateLogicalRouterStaticRoute mocks base method. +func (m *MockNbClient) UpdateLogicalRouterStaticRoute(route *ovnnb.LogicalRouterStaticRoute, fields ...any) error { + m.ctrl.T.Helper() + varargs := []any{route} + for _, a := range fields { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "UpdateLogicalRouterStaticRoute", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateLogicalRouterStaticRoute indicates an expected call of UpdateLogicalRouterStaticRoute. +func (mr *MockNbClientMockRecorder) UpdateLogicalRouterStaticRoute(route any, fields ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{route}, fields...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateLogicalRouterStaticRoute", reflect.TypeOf((*MockNbClient)(nil).UpdateLogicalRouterStaticRoute), varargs...) +} + // UpdateLogicalSwitchACL mocks base method. func (m *MockNbClient) UpdateLogicalSwitchACL(lsName, cidrBlock string, subnetAcls []v1.ACL, allowEWTraffic bool) error { m.ctrl.T.Helper() diff --git a/pkg/apis/kubeovn/v1/condition.go b/pkg/apis/kubeovn/v1/condition.go index 856869950ce..372391b01c1 100644 --- a/pkg/apis/kubeovn/v1/condition.go +++ b/pkg/apis/kubeovn/v1/condition.go @@ -5,6 +5,104 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +func (c *Conditions) addCondition(ctype ConditionType, status corev1.ConditionStatus, reason, message string, generation int64) { + if *c == nil { + *c = make(Conditions, 0, 1) + } + now := metav1.Now() + *c = append(*c, Condition{ + Type: ctype, + LastUpdateTime: now, + LastTransitionTime: now, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: generation, + }) +} + +// SetCondition creates or updates a condition +func (c *Conditions) SetCondition(ctype ConditionType, status corev1.ConditionStatus, reason, message string, generation int64) { + var v *Condition + for i := range *c { + if (*c)[i].Type == ctype { + v = &(*c)[i] + } + } + if v == nil { + c.addCondition(ctype, status, reason, message, generation) + return + } + + // check message ? + if v.Status == status && v.Reason == reason && v.Message == message && v.ObservedGeneration == generation { + return + } + + now := metav1.Now() + v.LastUpdateTime = now + if v.Status != status { + v.LastTransitionTime = now + } + v.Status = status + v.Reason = reason + v.Message = message + v.ObservedGeneration = generation +} + +// RemoveCondition removes the condition with the provided type. +func (c *Conditions) RemoveCondition(ctype ConditionType) { + for i, v := range *c { + if v.Type == ctype { + (*c)[i] = (*c)[len(*c)-1] + *c = (*c)[:len(*c)-1] + break + } + } +} + +// GetCondition get existing condition +func (c Conditions) GetCondition(ctype ConditionType) *Condition { + for _, v := range c { + if v.Type == ctype { + return v.DeepCopy() + } + } + return nil +} + +// IsConditionTrue - if condition is true +func (c Conditions) IsConditionTrue(ctype ConditionType, generation int64) bool { + v := c.GetCondition(ctype) + return v != nil && v.ObservedGeneration >= generation && v.Status == corev1.ConditionTrue +} + +func (c *Conditions) SetReady(reason string, generation int64) { + c.SetCondition(Ready, corev1.ConditionTrue, reason, "", generation) +} + +// IsReady returns true if ready condition is set +func (c Conditions) IsReady(generation int64) bool { + return c.IsConditionTrue(Ready, generation) +} + +func (c *Conditions) SetValidated(generation int64) { + c.SetCondition(Validated, corev1.ConditionTrue, "", "", generation) +} + +// IsValidated returns true if ready condition is set +func (c Conditions) IsValidated(generation int64) bool { + return c.IsConditionTrue(Validated, generation) +} + +// ConditionReason - return condition reason +func (c Conditions) ConditionReason(ctype ConditionType) string { + if v := c.GetCondition(ctype); v != nil { + return v.Reason + } + return "" +} + func (m *SubnetStatus) addCondition(ctype ConditionType, status corev1.ConditionStatus, reason, message string) { now := metav1.Now() c := &SubnetCondition{ diff --git a/pkg/apis/kubeovn/v1/register.go b/pkg/apis/kubeovn/v1/register.go index d974c1b3903..3730095dd5a 100644 --- a/pkg/apis/kubeovn/v1/register.go +++ b/pkg/apis/kubeovn/v1/register.go @@ -45,6 +45,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &VpcList{}, &VpcNatGateway{}, &VpcNatGatewayList{}, + &VpcEgressGateway{}, + &VpcEgressGatewayList{}, &Vip{}, &VipList{}, &IptablesEIP{}, diff --git a/pkg/apis/kubeovn/v1/types.go b/pkg/apis/kubeovn/v1/types.go index 19badbfe2f1..7d3381bfb16 100644 --- a/pkg/apis/kubeovn/v1/types.go +++ b/pkg/apis/kubeovn/v1/types.go @@ -70,10 +70,21 @@ const ( Validated = "Validated" // Error => last recorded error Error = "Error" + // Init => controller is initializing this resource + Init = "Init" ReasonInit = "Init" ) +// Phase represents resource phase +type Phase string + +const ( + PhasePending Phase = "Pending" + PhaseProcessing Phase = "Processing" + PhaseCompleted Phase = "Completed" +) + // +genclient // +genclient:noStatus // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -212,6 +223,10 @@ type Condition struct { // A human readable message indicating details about the transition. // +optional Message string `json:"message,omitempty"` + // ObservedGeneration represents the .metadata.generation that the condition was set based upon. + // For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, + // the condition is out of date with respect to the current state of the instance. + ObservedGeneration int64 `json:"observedGeneration,omitempty"` // Last time the condition was probed // +optional LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"` @@ -220,6 +235,8 @@ type Condition struct { LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` } +type Conditions []Condition + // SubnetCondition describes the state of an object at a certain point. // +k8s:deepcopy-gen=true type SubnetCondition Condition @@ -429,6 +446,14 @@ type VpcSpec struct { EnableExternal bool `json:"enableExternal,omitempty"` ExtraExternalSubnets []string `json:"extraExternalSubnets,omitempty"` EnableBfd bool `json:"enableBfd,omitempty"` + BFDPort *BFDPort `json:"bfdPort"` +} + +type BFDPort struct { + Enabled bool `json:"enabled"` + IP string `json:"ip,omitempty"` + + NodeSelector *metav1.LabelSelector `json:"nodeSelector,omitempty"` } type VpcPeering struct { @@ -491,6 +516,15 @@ type VpcStatus struct { EnableExternal bool `json:"enableExternal"` ExtraExternalSubnets []string `json:"extraExternalSubnets"` EnableBfd bool `json:"enableBfd"` + + BFDPort BFDPortStatus `json:"bfdPort"` +} + +type BFDPortStatus struct { + Enabled bool `json:"enabled"` + Name string `json:"name,omitempty"` + IP string `json:"ip,omitempty"` + Nodes []string `json:"nodes,omitempty"` } // VpcCondition describes the state of an object at a certain point. @@ -551,6 +585,82 @@ type VpcNatStatus struct { Affinity corev1.Affinity `json:"affinity" patchStrategy:"merge"` } +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +type VpcNatGatewayList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []VpcNatGateway `json:"items"` +} + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +resourceName=vpc-egress-gateways + +type VpcEgressGateway struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec VpcEgressGatewaySpec `json:"spec"` + Status VpcEgressGatewayStatus `json:"status,omitempty"` +} + +func (g *VpcEgressGateway) Ready() bool { + return g.Status.Conditions.IsReady(g.Generation) +} + +type VpcEgressGatewaySpec struct { + VPC string `json:"vpc,omitempty"` + Replicas int32 `json:"replicas"` + Image string `json:"image,omitempty"` + InternalSubnet string `json:"internalSubnet,omitempty"` + ExternalSubnet string `json:"externalSubnet"` + InternalIPs []string `json:"internalIPs,omitempty"` + ExternalIPs []string `json:"externalIPs,omitempty"` + + BFD VpcEgressGatewayBFDConfig `json:"bfd"` + Policies []VpcEgressGatewayPolicy `json:"policies,omitempty"` + NodeSelector []VpcEgressGatewayNodeSelector `json:"nodeSelector,omitempty"` +} + +type VpcEgressGatewayBFDConfig struct { + Enabled bool `json:"enabled"` + MinRX int `json:"minRX"` + MinTX int `json:"minTX"` + Multiplier int `json:"multiplier"` +} + +type VpcEgressGatewayPolicy struct { + SNAT bool `json:"snat"` + IPBlocks []string `json:"ipBlocks,omitempty"` + Subnets []string `json:"subnets,omitempty"` +} + +type VpcEgressGatewayNodeSelector struct { + MatchLabels map[string]string `json:"matchLabels,omitempty"` + MatchExpressions []corev1.NodeSelectorRequirement `json:"matchExpressions,omitempty"` + MatchFields []corev1.NodeSelectorRequirement `json:"matchFields,omitempty"` +} + +type VpcEgressGatewayStatus struct { + Ready bool `json:"ready"` + Phase Phase `json:"phase"` + Workload string `json:"workload,omitempty"` + InternalIPs []string `json:"internalIPs,omitempty"` + ExternalIPs []string `json:"externalIPs,omitempty"` + Conditions Conditions `json:"conditions,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +type VpcEgressGatewayList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []VpcEgressGateway `json:"items"` +} + // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +genclient:nonNamespaced @@ -747,15 +857,6 @@ type IptablesDnatRuleList struct { Items []IptablesDnatRule `json:"items"` } -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -type VpcNatGatewayList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata"` - - Items []VpcNatGateway `json:"items"` -} - // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +genclient:nonNamespaced diff --git a/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go b/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go index 14f37f4fa3d..ac4fca3cb9f 100644 --- a/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go +++ b/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go @@ -43,6 +43,48 @@ func (in *ACL) DeepCopy() *ACL { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BFDPort) DeepCopyInto(out *BFDPort) { + *out = *in + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = new(metav1.LabelSelector) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BFDPort. +func (in *BFDPort) DeepCopy() *BFDPort { + if in == nil { + return nil + } + out := new(BFDPort) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BFDPortStatus) DeepCopyInto(out *BFDPortStatus) { + *out = *in + if in.Nodes != nil { + in, out := &in.Nodes, &out.Nodes + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BFDPortStatus. +func (in *BFDPortStatus) DeepCopy() *BFDPortStatus { + if in == nil { + return nil + } + out := new(BFDPortStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Condition) DeepCopyInto(out *Condition) { *out = *in @@ -61,6 +103,28 @@ func (in *Condition) DeepCopy() *Condition { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in Conditions) DeepCopyInto(out *Conditions) { + { + in := &in + *out = make(Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + return + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Conditions. +func (in Conditions) DeepCopy() Conditions { + if in == nil { + return nil + } + out := new(Conditions) + in.DeepCopyInto(out) + return *out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CustomInterface) DeepCopyInto(out *CustomInterface) { *out = *in @@ -2544,6 +2608,220 @@ func (in *VpcDnsList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VpcEgressGateway) DeepCopyInto(out *VpcEgressGateway) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGateway. +func (in *VpcEgressGateway) DeepCopy() *VpcEgressGateway { + if in == nil { + return nil + } + out := new(VpcEgressGateway) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VpcEgressGateway) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VpcEgressGatewayBFDConfig) DeepCopyInto(out *VpcEgressGatewayBFDConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayBFDConfig. +func (in *VpcEgressGatewayBFDConfig) DeepCopy() *VpcEgressGatewayBFDConfig { + if in == nil { + return nil + } + out := new(VpcEgressGatewayBFDConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VpcEgressGatewayList) DeepCopyInto(out *VpcEgressGatewayList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]VpcEgressGateway, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayList. +func (in *VpcEgressGatewayList) DeepCopy() *VpcEgressGatewayList { + if in == nil { + return nil + } + out := new(VpcEgressGatewayList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VpcEgressGatewayList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VpcEgressGatewayNodeSelector) DeepCopyInto(out *VpcEgressGatewayNodeSelector) { + *out = *in + if in.MatchLabels != nil { + in, out := &in.MatchLabels, &out.MatchLabels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.MatchExpressions != nil { + in, out := &in.MatchExpressions, &out.MatchExpressions + *out = make([]corev1.NodeSelectorRequirement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.MatchFields != nil { + in, out := &in.MatchFields, &out.MatchFields + *out = make([]corev1.NodeSelectorRequirement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayNodeSelector. +func (in *VpcEgressGatewayNodeSelector) DeepCopy() *VpcEgressGatewayNodeSelector { + if in == nil { + return nil + } + out := new(VpcEgressGatewayNodeSelector) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VpcEgressGatewayPolicy) DeepCopyInto(out *VpcEgressGatewayPolicy) { + *out = *in + if in.IPBlocks != nil { + in, out := &in.IPBlocks, &out.IPBlocks + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Subnets != nil { + in, out := &in.Subnets, &out.Subnets + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayPolicy. +func (in *VpcEgressGatewayPolicy) DeepCopy() *VpcEgressGatewayPolicy { + if in == nil { + return nil + } + out := new(VpcEgressGatewayPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VpcEgressGatewaySpec) DeepCopyInto(out *VpcEgressGatewaySpec) { + *out = *in + if in.InternalIPs != nil { + in, out := &in.InternalIPs, &out.InternalIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ExternalIPs != nil { + in, out := &in.ExternalIPs, &out.ExternalIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } + out.BFD = in.BFD + if in.Policies != nil { + in, out := &in.Policies, &out.Policies + *out = make([]VpcEgressGatewayPolicy, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make([]VpcEgressGatewayNodeSelector, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewaySpec. +func (in *VpcEgressGatewaySpec) DeepCopy() *VpcEgressGatewaySpec { + if in == nil { + return nil + } + out := new(VpcEgressGatewaySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VpcEgressGatewayStatus) DeepCopyInto(out *VpcEgressGatewayStatus) { + *out = *in + if in.InternalIPs != nil { + in, out := &in.InternalIPs, &out.InternalIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ExternalIPs != nil { + in, out := &in.ExternalIPs, &out.ExternalIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcEgressGatewayStatus. +func (in *VpcEgressGatewayStatus) DeepCopy() *VpcEgressGatewayStatus { + if in == nil { + return nil + } + out := new(VpcEgressGatewayStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VpcList) DeepCopyInto(out *VpcList) { *out = *in @@ -2769,6 +3047,11 @@ func (in *VpcSpec) DeepCopyInto(out *VpcSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.BFDPort != nil { + in, out := &in.BFDPort, &out.BFDPort + *out = new(BFDPort) + (*in).DeepCopyInto(*out) + } return } @@ -2807,6 +3090,7 @@ func (in *VpcStatus) DeepCopyInto(out *VpcStatus) { *out = make([]string, len(*in)) copy(*out, *in) } + in.BFDPort.DeepCopyInto(&out.BFDPort) return } diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go index be653ef62ab..a4857aada11 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go @@ -104,6 +104,10 @@ func (c *FakeKubeovnV1) VpcDnses() v1.VpcDnsInterface { return &FakeVpcDnses{c} } +func (c *FakeKubeovnV1) VpcEgressGateways(namespace string) v1.VpcEgressGatewayInterface { + return &FakeVpcEgressGateways{c, namespace} +} + func (c *FakeKubeovnV1) VpcNatGateways() v1.VpcNatGatewayInterface { return &FakeVpcNatGateways{c} } diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_vpcegressgateway.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_vpcegressgateway.go new file mode 100644 index 00000000000..013cca6d5f7 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_vpcegressgateway.go @@ -0,0 +1,147 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeVpcEgressGateways implements VpcEgressGatewayInterface +type FakeVpcEgressGateways struct { + Fake *FakeKubeovnV1 + ns string +} + +var vpcegressgatewaysResource = v1.SchemeGroupVersion.WithResource("vpc-egress-gateways") + +var vpcegressgatewaysKind = v1.SchemeGroupVersion.WithKind("VpcEgressGateway") + +// Get takes name of the vpcEgressGateway, and returns the corresponding vpcEgressGateway object, and an error if there is any. +func (c *FakeVpcEgressGateways) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.VpcEgressGateway, err error) { + emptyResult := &v1.VpcEgressGateway{} + obj, err := c.Fake. + Invokes(testing.NewGetActionWithOptions(vpcegressgatewaysResource, c.ns, name, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1.VpcEgressGateway), err +} + +// List takes label and field selectors, and returns the list of VpcEgressGateways that match those selectors. +func (c *FakeVpcEgressGateways) List(ctx context.Context, opts metav1.ListOptions) (result *v1.VpcEgressGatewayList, err error) { + emptyResult := &v1.VpcEgressGatewayList{} + obj, err := c.Fake. + Invokes(testing.NewListActionWithOptions(vpcegressgatewaysResource, vpcegressgatewaysKind, c.ns, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1.VpcEgressGatewayList{ListMeta: obj.(*v1.VpcEgressGatewayList).ListMeta} + for _, item := range obj.(*v1.VpcEgressGatewayList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested vpcEgressGateways. +func (c *FakeVpcEgressGateways) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchActionWithOptions(vpcegressgatewaysResource, c.ns, opts)) + +} + +// Create takes the representation of a vpcEgressGateway and creates it. Returns the server's representation of the vpcEgressGateway, and an error, if there is any. +func (c *FakeVpcEgressGateways) Create(ctx context.Context, vpcEgressGateway *v1.VpcEgressGateway, opts metav1.CreateOptions) (result *v1.VpcEgressGateway, err error) { + emptyResult := &v1.VpcEgressGateway{} + obj, err := c.Fake. + Invokes(testing.NewCreateActionWithOptions(vpcegressgatewaysResource, c.ns, vpcEgressGateway, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1.VpcEgressGateway), err +} + +// Update takes the representation of a vpcEgressGateway and updates it. Returns the server's representation of the vpcEgressGateway, and an error, if there is any. +func (c *FakeVpcEgressGateways) Update(ctx context.Context, vpcEgressGateway *v1.VpcEgressGateway, opts metav1.UpdateOptions) (result *v1.VpcEgressGateway, err error) { + emptyResult := &v1.VpcEgressGateway{} + obj, err := c.Fake. + Invokes(testing.NewUpdateActionWithOptions(vpcegressgatewaysResource, c.ns, vpcEgressGateway, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1.VpcEgressGateway), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeVpcEgressGateways) UpdateStatus(ctx context.Context, vpcEgressGateway *v1.VpcEgressGateway, opts metav1.UpdateOptions) (result *v1.VpcEgressGateway, err error) { + emptyResult := &v1.VpcEgressGateway{} + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceActionWithOptions(vpcegressgatewaysResource, "status", c.ns, vpcEgressGateway, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1.VpcEgressGateway), err +} + +// Delete takes name of the vpcEgressGateway and deletes it. Returns an error if one occurs. +func (c *FakeVpcEgressGateways) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(vpcegressgatewaysResource, c.ns, name, opts), &v1.VpcEgressGateway{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeVpcEgressGateways) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error { + action := testing.NewDeleteCollectionActionWithOptions(vpcegressgatewaysResource, c.ns, opts, listOpts) + + _, err := c.Fake.Invokes(action, &v1.VpcEgressGatewayList{}) + return err +} + +// Patch applies the patch and returns the patched vpcEgressGateway. +func (c *FakeVpcEgressGateways) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.VpcEgressGateway, err error) { + emptyResult := &v1.VpcEgressGateway{} + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceActionWithOptions(vpcegressgatewaysResource, c.ns, name, pt, data, opts, subresources...), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1.VpcEgressGateway), err +} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go index 15876f7e55b..a4a3dcf75f0 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go @@ -56,4 +56,6 @@ type VpcExpansion interface{} type VpcDnsExpansion interface{} +type VpcEgressGatewayExpansion interface{} + type VpcNatGatewayExpansion interface{} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go index bd9b017ea8c..aa73b0f4b66 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go @@ -47,6 +47,7 @@ type KubeovnV1Interface interface { VlansGetter VpcsGetter VpcDnsesGetter + VpcEgressGatewaysGetter VpcNatGatewaysGetter } @@ -131,6 +132,10 @@ func (c *KubeovnV1Client) VpcDnses() VpcDnsInterface { return newVpcDnses(c) } +func (c *KubeovnV1Client) VpcEgressGateways(namespace string) VpcEgressGatewayInterface { + return newVpcEgressGateways(c, namespace) +} + func (c *KubeovnV1Client) VpcNatGateways() VpcNatGatewayInterface { return newVpcNatGateways(c) } diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/vpcegressgateway.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/vpcegressgateway.go new file mode 100644 index 00000000000..09f4bafa669 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/vpcegressgateway.go @@ -0,0 +1,69 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1 + +import ( + "context" + + v1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + scheme "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/scheme" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// VpcEgressGatewaysGetter has a method to return a VpcEgressGatewayInterface. +// A group's client should implement this interface. +type VpcEgressGatewaysGetter interface { + VpcEgressGateways(namespace string) VpcEgressGatewayInterface +} + +// VpcEgressGatewayInterface has methods to work with VpcEgressGateway resources. +type VpcEgressGatewayInterface interface { + Create(ctx context.Context, vpcEgressGateway *v1.VpcEgressGateway, opts metav1.CreateOptions) (*v1.VpcEgressGateway, error) + Update(ctx context.Context, vpcEgressGateway *v1.VpcEgressGateway, opts metav1.UpdateOptions) (*v1.VpcEgressGateway, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, vpcEgressGateway *v1.VpcEgressGateway, opts metav1.UpdateOptions) (*v1.VpcEgressGateway, error) + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error + Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.VpcEgressGateway, error) + List(ctx context.Context, opts metav1.ListOptions) (*v1.VpcEgressGatewayList, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.VpcEgressGateway, err error) + VpcEgressGatewayExpansion +} + +// vpcEgressGateways implements VpcEgressGatewayInterface +type vpcEgressGateways struct { + *gentype.ClientWithList[*v1.VpcEgressGateway, *v1.VpcEgressGatewayList] +} + +// newVpcEgressGateways returns a VpcEgressGateways +func newVpcEgressGateways(c *KubeovnV1Client, namespace string) *vpcEgressGateways { + return &vpcEgressGateways{ + gentype.NewClientWithList[*v1.VpcEgressGateway, *v1.VpcEgressGatewayList]( + "vpc-egress-gateways", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *v1.VpcEgressGateway { return &v1.VpcEgressGateway{} }, + func() *v1.VpcEgressGatewayList { return &v1.VpcEgressGatewayList{} }), + } +} diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go index 61c4b5fc9fa..03bcdd8f8ef 100644 --- a/pkg/client/informers/externalversions/generic.go +++ b/pkg/client/informers/externalversions/generic.go @@ -91,6 +91,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().Vpcs().Informer()}, nil case v1.SchemeGroupVersion.WithResource("vpc-dnses"): return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().VpcDnses().Informer()}, nil + case v1.SchemeGroupVersion.WithResource("vpc-egress-gateways"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().VpcEgressGateways().Informer()}, nil case v1.SchemeGroupVersion.WithResource("vpc-nat-gateways"): return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().VpcNatGateways().Informer()}, nil diff --git a/pkg/client/informers/externalversions/kubeovn/v1/interface.go b/pkg/client/informers/externalversions/kubeovn/v1/interface.go index 2b13d05651d..69cad95925a 100644 --- a/pkg/client/informers/externalversions/kubeovn/v1/interface.go +++ b/pkg/client/informers/externalversions/kubeovn/v1/interface.go @@ -62,6 +62,8 @@ type Interface interface { Vpcs() VpcInformer // VpcDnses returns a VpcDnsInformer. VpcDnses() VpcDnsInformer + // VpcEgressGateways returns a VpcEgressGatewayInformer. + VpcEgressGateways() VpcEgressGatewayInformer // VpcNatGateways returns a VpcNatGatewayInformer. VpcNatGateways() VpcNatGatewayInformer } @@ -172,6 +174,11 @@ func (v *version) VpcDnses() VpcDnsInformer { return &vpcDnsInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} } +// VpcEgressGateways returns a VpcEgressGatewayInformer. +func (v *version) VpcEgressGateways() VpcEgressGatewayInformer { + return &vpcEgressGatewayInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + // VpcNatGateways returns a VpcNatGatewayInformer. func (v *version) VpcNatGateways() VpcNatGatewayInformer { return &vpcNatGatewayInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} diff --git a/pkg/client/informers/externalversions/kubeovn/v1/vpcegressgateway.go b/pkg/client/informers/externalversions/kubeovn/v1/vpcegressgateway.go new file mode 100644 index 00000000000..aa8f3e94067 --- /dev/null +++ b/pkg/client/informers/externalversions/kubeovn/v1/vpcegressgateway.go @@ -0,0 +1,90 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + "context" + time "time" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + versioned "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" + internalinterfaces "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions/internalinterfaces" + v1 "github.com/kubeovn/kube-ovn/pkg/client/listers/kubeovn/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// VpcEgressGatewayInformer provides access to a shared informer and lister for +// VpcEgressGateways. +type VpcEgressGatewayInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1.VpcEgressGatewayLister +} + +type vpcEgressGatewayInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewVpcEgressGatewayInformer constructs a new informer for VpcEgressGateway type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewVpcEgressGatewayInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredVpcEgressGatewayInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredVpcEgressGatewayInformer constructs a new informer for VpcEgressGateway type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredVpcEgressGatewayInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.KubeovnV1().VpcEgressGateways(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.KubeovnV1().VpcEgressGateways(namespace).Watch(context.TODO(), options) + }, + }, + &kubeovnv1.VpcEgressGateway{}, + resyncPeriod, + indexers, + ) +} + +func (f *vpcEgressGatewayInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredVpcEgressGatewayInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *vpcEgressGatewayInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&kubeovnv1.VpcEgressGateway{}, f.defaultInformer) +} + +func (f *vpcEgressGatewayInformer) Lister() v1.VpcEgressGatewayLister { + return v1.NewVpcEgressGatewayLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/listers/kubeovn/v1/expansion_generated.go b/pkg/client/listers/kubeovn/v1/expansion_generated.go index 998c1ddaa14..b474d11b77d 100644 --- a/pkg/client/listers/kubeovn/v1/expansion_generated.go +++ b/pkg/client/listers/kubeovn/v1/expansion_generated.go @@ -94,6 +94,14 @@ type VpcListerExpansion interface{} // VpcDnsLister. type VpcDnsListerExpansion interface{} +// VpcEgressGatewayListerExpansion allows custom methods to be added to +// VpcEgressGatewayLister. +type VpcEgressGatewayListerExpansion interface{} + +// VpcEgressGatewayNamespaceListerExpansion allows custom methods to be added to +// VpcEgressGatewayNamespaceLister. +type VpcEgressGatewayNamespaceListerExpansion interface{} + // VpcNatGatewayListerExpansion allows custom methods to be added to // VpcNatGatewayLister. type VpcNatGatewayListerExpansion interface{} diff --git a/pkg/client/listers/kubeovn/v1/vpcegressgateway.go b/pkg/client/listers/kubeovn/v1/vpcegressgateway.go new file mode 100644 index 00000000000..db9295e3149 --- /dev/null +++ b/pkg/client/listers/kubeovn/v1/vpcegressgateway.go @@ -0,0 +1,70 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +import ( + v1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/listers" + "k8s.io/client-go/tools/cache" +) + +// VpcEgressGatewayLister helps list VpcEgressGateways. +// All objects returned here must be treated as read-only. +type VpcEgressGatewayLister interface { + // List lists all VpcEgressGateways in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1.VpcEgressGateway, err error) + // VpcEgressGateways returns an object that can list and get VpcEgressGateways. + VpcEgressGateways(namespace string) VpcEgressGatewayNamespaceLister + VpcEgressGatewayListerExpansion +} + +// vpcEgressGatewayLister implements the VpcEgressGatewayLister interface. +type vpcEgressGatewayLister struct { + listers.ResourceIndexer[*v1.VpcEgressGateway] +} + +// NewVpcEgressGatewayLister returns a new VpcEgressGatewayLister. +func NewVpcEgressGatewayLister(indexer cache.Indexer) VpcEgressGatewayLister { + return &vpcEgressGatewayLister{listers.New[*v1.VpcEgressGateway](indexer, v1.Resource("vpcegressgateway"))} +} + +// VpcEgressGateways returns an object that can list and get VpcEgressGateways. +func (s *vpcEgressGatewayLister) VpcEgressGateways(namespace string) VpcEgressGatewayNamespaceLister { + return vpcEgressGatewayNamespaceLister{listers.NewNamespaced[*v1.VpcEgressGateway](s.ResourceIndexer, namespace)} +} + +// VpcEgressGatewayNamespaceLister helps list and get VpcEgressGateways. +// All objects returned here must be treated as read-only. +type VpcEgressGatewayNamespaceLister interface { + // List lists all VpcEgressGateways in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1.VpcEgressGateway, err error) + // Get retrieves the VpcEgressGateway from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1.VpcEgressGateway, error) + VpcEgressGatewayNamespaceListerExpansion +} + +// vpcEgressGatewayNamespaceLister implements the VpcEgressGatewayNamespaceLister +// interface. +type vpcEgressGatewayNamespaceLister struct { + listers.ResourceIndexer[*v1.VpcEgressGateway] +} diff --git a/pkg/controller/config.go b/pkg/controller/config.go index 9963e31445d..e98c14b3a29 100644 --- a/pkg/controller/config.go +++ b/pkg/controller/config.go @@ -109,6 +109,9 @@ type Configuration struct { BfdDetectMult int NodeLocalDNSIPs []string + + // used to set vpc-egress-gateway image + Image string } // ParseFlags parses cmd args then init kubeclient and conf @@ -186,6 +189,8 @@ func ParseFlags() (*Configuration, error) { argBfdMinTx = pflag.Int("bfd-min-tx", 100, "This is the minimum interval, in milliseconds, ovn would like to use when transmitting BFD Control packets") argBfdMinRx = pflag.Int("bfd-min-rx", 100, "This is the minimum interval, in milliseconds, between received BFD Control packets") argBfdDetectMult = pflag.Int("detect-mult", 3, "The negotiated transmit interval, multiplied by this value, provides the Detection Time for the receiving system in Asynchronous mode.") + + argImage = pflag.String("image", "", "The image for vpc-egress-gateway") ) klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) @@ -269,6 +274,7 @@ func ParseFlags() (*Configuration, error) { BfdMinRx: *argBfdMinRx, BfdDetectMult: *argBfdDetectMult, EnableANP: *argEnableANP, + Image: *argImage, } if config.NetworkType == util.NetworkTypeVlan && config.DefaultHostInterface == "" { diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 7146382f554..14e7a1886a1 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -17,6 +17,7 @@ import ( kubeinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/scheme" typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + appsv1 "k8s.io/client-go/listers/apps/v1" certListerv1 "k8s.io/client-go/listers/certificates/v1" v1 "k8s.io/client-go/listers/core/v1" netv1 "k8s.io/client-go/listers/networking/v1" @@ -95,6 +96,12 @@ type Controller struct { updateVpcSubnetQueue workqueue.TypedRateLimitingInterface[string] vpcNatGwKeyMutex keymutex.KeyMutex + vpcEgressGatewayLister kubeovnlister.VpcEgressGatewayLister + vpcEgressGatewaySynced cache.InformerSynced + addOrUpdateVpcEgressGatewayQueue workqueue.TypedRateLimitingInterface[string] + delVpcEgressGatewayQueue workqueue.TypedRateLimitingInterface[string] + vpcEgressGatewayKeyMutex keymutex.KeyMutex + switchLBRuleLister kubeovnlister.SwitchLBRuleLister switchLBRuleSynced cache.InformerSynced addSwitchLBRuleQueue workqueue.TypedRateLimitingInterface[string] @@ -218,6 +225,9 @@ type Controller struct { addOrUpdateEndpointQueue workqueue.TypedRateLimitingInterface[string] epKeyMutex keymutex.KeyMutex + deploymentsLister appsv1.DeploymentLister + deploymentsSynced cache.InformerSynced + npsLister netv1.NetworkPolicyLister npsSynced cache.InformerSynced updateNpQueue workqueue.TypedRateLimitingInterface[string] @@ -303,6 +313,7 @@ func Run(ctx context.Context, config *Configuration) { vpcInformer := kubeovnInformerFactory.Kubeovn().V1().Vpcs() vpcNatGatewayInformer := kubeovnInformerFactory.Kubeovn().V1().VpcNatGateways() + vpcEgressGatewayInformer := kubeovnInformerFactory.Kubeovn().V1().VpcEgressGateways() subnetInformer := kubeovnInformerFactory.Kubeovn().V1().Subnets() ippoolInformer := kubeovnInformerFactory.Kubeovn().V1().IPPools() ipInformer := kubeovnInformerFactory.Kubeovn().V1().IPs() @@ -319,6 +330,7 @@ func Run(ctx context.Context, config *Configuration) { nodeInformer := informerFactory.Core().V1().Nodes() serviceInformer := informerFactory.Core().V1().Services() endpointInformer := informerFactory.Core().V1().Endpoints() + deploymentInformer := informerFactory.Apps().V1().Deployments() qosPolicyInformer := kubeovnInformerFactory.Kubeovn().V1().QoSPolicies() configMapInformer := cmInformerFactory.Core().V1().ConfigMaps() npInformer := informerFactory.Networking().V1().NetworkPolicies() @@ -362,6 +374,12 @@ func Run(ctx context.Context, config *Configuration) { updateVpcSubnetQueue: newTypedRateLimitingQueue("UpdateVpcSubnet", custCrdRateLimiter), vpcNatGwKeyMutex: keymutex.NewHashed(numKeyLocks), + vpcEgressGatewayLister: vpcEgressGatewayInformer.Lister(), + vpcEgressGatewaySynced: vpcEgressGatewayInformer.Informer().HasSynced, + addOrUpdateVpcEgressGatewayQueue: newTypedRateLimitingQueue("AddOrUpdateVpcEgressGateway", custCrdRateLimiter), + delVpcEgressGatewayQueue: newTypedRateLimitingQueue("DeleteVpcEgressGateway", custCrdRateLimiter), + vpcEgressGatewayKeyMutex: keymutex.NewHashed(numKeyLocks), + subnetsLister: subnetInformer.Lister(), subnetSynced: subnetInformer.Informer().HasSynced, addOrUpdateSubnetQueue: newTypedRateLimitingQueue[string]("AddSubnet", nil), @@ -462,6 +480,9 @@ func Run(ctx context.Context, config *Configuration) { addOrUpdateEndpointQueue: newTypedRateLimitingQueue[string]("UpdateEndpoint", nil), epKeyMutex: keymutex.NewHashed(numKeyLocks), + deploymentsLister: deploymentInformer.Lister(), + deploymentsSynced: deploymentInformer.Informer().HasSynced, + qosPoliciesLister: qosPolicyInformer.Lister(), qosPolicySynced: qosPolicyInformer.Informer().HasSynced, addQoSPolicyQueue: newTypedRateLimitingQueue("AddQoSPolicy", custCrdRateLimiter), @@ -593,11 +614,12 @@ func Run(ctx context.Context, config *Configuration) { klog.Info("Waiting for informer caches to sync") cacheSyncs := []cache.InformerSynced{ - controller.vpcNatGatewaySynced, controller.vpcSynced, controller.subnetSynced, + controller.vpcNatGatewaySynced, controller.vpcEgressGatewaySynced, + controller.vpcSynced, controller.subnetSynced, controller.ipSynced, controller.virtualIpsSynced, controller.iptablesEipSynced, controller.iptablesFipSynced, controller.iptablesDnatRuleSynced, controller.iptablesSnatRuleSynced, controller.vlanSynced, controller.podsSynced, controller.namespacesSynced, controller.nodesSynced, - controller.serviceSynced, controller.endpointsSynced, controller.configMapsSynced, + controller.serviceSynced, controller.endpointsSynced, controller.deploymentsSynced, controller.configMapsSynced, controller.ovnEipSynced, controller.ovnFipSynced, controller.ovnSnatRuleSynced, controller.ovnDnatRuleSynced, } @@ -653,6 +675,13 @@ func Run(ctx context.Context, config *Configuration) { util.LogFatalAndExit(err, "failed to add endpoint event handler") } + if _, err = deploymentInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: controller.enqueueAddDeployment, + UpdateFunc: controller.enqueueUpdateDeployment, + }); err != nil { + util.LogFatalAndExit(err, "failed to add deployment event handler") + } + if _, err = vpcInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: controller.enqueueAddVpc, UpdateFunc: controller.enqueueUpdateVpc, @@ -669,6 +698,14 @@ func Run(ctx context.Context, config *Configuration) { util.LogFatalAndExit(err, "failed to add vpc nat gateway event handler") } + if _, err = vpcEgressGatewayInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: controller.enqueueAddVpcEgressGateway, + UpdateFunc: controller.enqueueUpdateVpcEgressGateway, + DeleteFunc: controller.enqueueDeleteVpcEgressGateway, + }); err != nil { + util.LogFatalAndExit(err, "failed to add vpc egress gateway event handler") + } + if _, err = subnetInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: controller.enqueueAddSubnet, UpdateFunc: controller.enqueueUpdateSubnet, @@ -964,6 +1001,9 @@ func (c *Controller) shutdown() { c.updateVpcSnatQueue.ShutDown() c.updateVpcSubnetQueue.ShutDown() + c.addOrUpdateVpcEgressGatewayQueue.ShutDown() + c.delVpcEgressGatewayQueue.ShutDown() + if c.config.EnableLb { c.addSwitchLBRuleQueue.ShutDown() c.delSwitchLBRuleQueue.ShutDown() @@ -1045,10 +1085,14 @@ func (c *Controller) startWorkers(ctx context.Context) { klog.Info("Starting workers") go wait.Until(runWorker("add/update vpc", c.addOrUpdateVpcQueue, c.handleAddOrUpdateVpc), time.Second, ctx.Done()) + go wait.Until(runWorker("delete vpc", c.delVpcQueue, c.handleDelVpc), time.Second, ctx.Done()) + go wait.Until(runWorker("update status of vpc", c.updateVpcStatusQueue, c.handleUpdateVpcStatus), time.Second, ctx.Done()) go wait.Until(runWorker("add/update vpc nat gateway", c.addOrUpdateVpcNatGatewayQueue, c.handleAddOrUpdateVpcNatGw), time.Second, ctx.Done()) go wait.Until(runWorker("init vpc nat gateway", c.initVpcNatGatewayQueue, c.handleInitVpcNatGw), time.Second, ctx.Done()) go wait.Until(runWorker("delete vpc nat gateway", c.delVpcNatGatewayQueue, c.handleDelVpcNatGw), time.Second, ctx.Done()) + go wait.Until(runWorker("add/update vpc egress gateway", c.addOrUpdateVpcEgressGatewayQueue, c.handleAddOrUpdateVpcEgressGateway), time.Second, ctx.Done()) + go wait.Until(runWorker("delete vpc egress gateway", c.delVpcEgressGatewayQueue, c.handleDelVpcEgressGateway), time.Second, ctx.Done()) go wait.Until(runWorker("update fip for vpc nat gateway", c.updateVpcFloatingIPQueue, c.handleUpdateVpcFloatingIP), time.Second, ctx.Done()) go wait.Until(runWorker("update eip for vpc nat gateway", c.updateVpcEipQueue, c.handleUpdateVpcEip), time.Second, ctx.Done()) go wait.Until(runWorker("update dnat for vpc nat gateway", c.updateVpcDnatQueue, c.handleUpdateVpcDnat), time.Second, ctx.Done()) @@ -1100,9 +1144,6 @@ func (c *Controller) startWorkers(ctx context.Context) { } } - go wait.Until(runWorker("delete vpc", c.delVpcQueue, c.handleDelVpc), time.Second, ctx.Done()) - go wait.Until(runWorker("update status of vpc", c.updateVpcStatusQueue, c.handleUpdateVpcStatus), time.Second, ctx.Done()) - if c.config.EnableLb { go wait.Until(runWorker("add service", c.addServiceQueue, c.handleAddService), time.Second, ctx.Done()) // run in a single worker to avoid delete the last vip, which will lead ovn to delete the loadbalancer diff --git a/pkg/controller/deployment.go b/pkg/controller/deployment.go new file mode 100644 index 00000000000..3986812378a --- /dev/null +++ b/pkg/controller/deployment.go @@ -0,0 +1,44 @@ +package controller + +import ( + "fmt" + "reflect" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/klog/v2" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" +) + +var ( + vpcEgressGatewayGroupVersion string + vpcEgressGatewayKind string +) + +func init() { + name := reflect.TypeOf(&kubeovnv1.VpcEgressGateway{}).Elem().Name() + gvk := kubeovnv1.SchemeGroupVersion.WithKind(name) + vpcEgressGatewayGroupVersion = gvk.GroupVersion().String() + vpcEgressGatewayKind = gvk.Kind +} + +func (c *Controller) enqueueAddDeployment(obj interface{}) { + deploy := obj.(*appsv1.Deployment) + var vegName string + for _, ref := range deploy.OwnerReferences { + if ref.APIVersion == vpcEgressGatewayGroupVersion && ref.Kind == vpcEgressGatewayKind { + vegName = ref.Name + break + } + } + + if vegName != "" { + key := fmt.Sprintf("%s/%s", deploy.Namespace, vegName) + klog.V(3).Infof("enqueue update vpc-egress-gateway %s", key) + c.addOrUpdateVpcEgressGatewayQueue.Add(key) + } +} + +func (c *Controller) enqueueUpdateDeployment(_, newObj interface{}) { + c.enqueueAddDeployment(newObj) +} diff --git a/pkg/controller/subnet.go b/pkg/controller/subnet.go index 2a1a4193641..7c506f252ff 100644 --- a/pkg/controller/subnet.go +++ b/pkg/controller/subnet.go @@ -1300,7 +1300,7 @@ func (c *Controller) reconcileCustomVpcBfdStaticRoute(vpcName, subnetName string klog.Error(err) return err } - bfd, err := c.OVNNbClient.CreateBFD(lrpEipName, eip.Status.V4Ip, c.config.BfdMinRx, c.config.BfdMinTx, c.config.BfdDetectMult) + bfd, err := c.OVNNbClient.CreateBFD(lrpEipName, eip.Status.V4Ip, c.config.BfdMinRx, c.config.BfdMinTx, c.config.BfdDetectMult, nil) if err != nil { klog.Error(err) return err diff --git a/pkg/controller/vpc.go b/pkg/controller/vpc.go index a0dc0d2c59b..ae04ae25b34 100644 --- a/pkg/controller/vpc.go +++ b/pkg/controller/vpc.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "math" "net" "reflect" "slices" @@ -43,6 +44,16 @@ func (c *Controller) enqueueAddVpc(obj interface{}) { } } +func vpcBFDPortChanged(oldObj, newObj *kubeovnv1.BFDPort) bool { + if oldObj == nil && newObj == nil { + return false + } + if oldObj == nil || newObj == nil { + return true + } + return oldObj.Enabled != newObj.Enabled || oldObj.IP != newObj.IP || !reflect.DeepEqual(oldObj.NodeSelector, newObj.NodeSelector) +} + func (c *Controller) enqueueUpdateVpc(oldObj, newObj interface{}) { oldVpc := oldObj.(*kubeovnv1.Vpc) newVpc := newObj.(*kubeovnv1.Vpc) @@ -56,14 +67,11 @@ func (c *Controller) enqueueUpdateVpc(oldObj, newObj interface{}) { !reflect.DeepEqual(oldVpc.Spec.ExtraExternalSubnets, newVpc.Spec.ExtraExternalSubnets) || oldVpc.Spec.EnableExternal != newVpc.Spec.EnableExternal || oldVpc.Spec.EnableBfd != newVpc.Spec.EnableBfd || + vpcBFDPortChanged(oldVpc.Spec.BFDPort, newVpc.Spec.BFDPort) || oldVpc.Labels[util.VpcExternalLabel] != newVpc.Labels[util.VpcExternalLabel] { // TODO:// label VpcExternalLabel replace with spec enable external - var ( - key string - err error - ) - - if key, err = cache.MetaNamespaceKeyFunc(newObj); err != nil { + key, err := cache.MetaNamespaceKeyFunc(newObj) + if err != nil { utilruntime.HandleError(err) return } @@ -249,13 +257,7 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { defer func() { _ = c.vpcKeyMutex.UnlockKey(key) }() klog.Infof("handle add/update vpc %s", key) - // get latest vpc info - var ( - vpc, cachedVpc *kubeovnv1.Vpc - err error - ) - - cachedVpc, err = c.vpcsLister.Get(key) + cachedVpc, err := c.vpcsLister.Get(key) if err != nil { if k8serrors.IsNotFound(err) { return nil @@ -263,9 +265,9 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { klog.Error(err) return err } - vpc = cachedVpc.DeepCopy() - if vpc, err = c.formatVpc(vpc); err != nil { + vpc, err := c.formatVpc(cachedVpc.DeepCopy()) + if err != nil { klog.Errorf("failed to format vpc %s: %v", key, err) return err } @@ -487,7 +489,7 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { // add new policies for _, item := range policyRouteNeedAdd { klog.Infof("add policy route for router: %s, match %s, action %s, nexthop %s, externalID %v", c.config.ClusterRouter, item.Match, string(item.Action), item.NextHopIP, externalIDs) - if err = c.OVNNbClient.AddLogicalRouterPolicy(vpc.Name, item.Priority, item.Match, string(item.Action), []string{item.NextHopIP}, externalIDs); err != nil { + if err = c.OVNNbClient.AddLogicalRouterPolicy(vpc.Name, item.Priority, item.Match, string(item.Action), []string{item.NextHopIP}, nil, externalIDs); err != nil { klog.Errorf("add policy route to vpc %s failed, %v", vpc.Name, err) return err } @@ -571,7 +573,7 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { lrpEipName := fmt.Sprintf("%s-%s", key, c.config.ExternalGatewaySwitch) v4ExtGw, _ := util.SplitStringIP(externalSubnet.Spec.Gateway) // TODO: dualstack - if _, err := c.OVNNbClient.CreateBFD(lrpEipName, v4ExtGw, c.config.BfdMinRx, c.config.BfdMinTx, c.config.BfdDetectMult); err != nil { + if _, err := c.OVNNbClient.CreateBFD(lrpEipName, v4ExtGw, c.config.BfdMinRx, c.config.BfdMinTx, c.config.BfdDetectMult, nil); err != nil { klog.Error(err) return err } @@ -617,7 +619,7 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { if !cachedVpc.Spec.EnableBfd && cachedVpc.Status.EnableBfd { lrpEipName := fmt.Sprintf("%s-%s", key, c.config.ExternalGatewaySwitch) - if err := c.OVNNbClient.DeleteBFD(lrpEipName, ""); err != nil { + if err := c.OVNNbClient.DeleteBFDByDstIP(lrpEipName, ""); err != nil { klog.Error(err) return err } @@ -651,9 +653,109 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { } } + bfdPortName, bfdPortNodes, err := c.reconcileVpcBfdLRP(vpc) + if err != nil { + klog.Error(err) + return err + } + if vpc.Spec.BFDPort == nil || !vpc.Spec.BFDPort.Enabled { + vpc.Status.BFDPort = kubeovnv1.BFDPortStatus{Enabled: false} + } else { + vpc.Status.BFDPort = kubeovnv1.BFDPortStatus{ + Enabled: true, + Name: bfdPortName, + IP: vpc.Spec.BFDPort.IP, + Nodes: bfdPortNodes, + } + } + if _, err = c.config.KubeOvnClient.KubeovnV1().Vpcs(). + UpdateStatus(context.Background(), vpc, metav1.UpdateOptions{}); err != nil { + klog.Error(err) + return err + } + return nil } +func (c *Controller) reconcileVpcBfdLRP(vpc *kubeovnv1.Vpc) (string, []string, error) { + portName := "bfd@" + vpc.Name + if vpc.Spec.BFDPort == nil || !vpc.Spec.BFDPort.Enabled { + if err := c.OVNNbClient.DeleteLogicalRouterPort(portName); err != nil { + err = fmt.Errorf("failed to delete BFD LRP %s: %w", portName, err) + klog.Error(err) + return portName, nil, err + } + if err := c.OVNNbClient.DeleteHAChassisGroup(portName); err != nil { + err = fmt.Errorf("failed to delete HA chassis group %s: %w", portName, err) + klog.Error(err) + return portName, nil, err + } + return portName, nil, nil + } + + var err error + chassisCount := 3 + selector := labels.Everything() + if vpc.Spec.BFDPort.NodeSelector != nil { + chassisCount = math.MaxInt + if selector, err = metav1.LabelSelectorAsSelector(vpc.Spec.BFDPort.NodeSelector); err != nil { + err = fmt.Errorf("failed to parse node selector %q: %w", vpc.Spec.BFDPort.NodeSelector.String(), err) + klog.Error(err) + return portName, nil, err + } + } + + nodes, err := c.nodesLister.List(selector) + if err != nil { + err = fmt.Errorf("failed to list nodes with selector %q: %w", vpc.Spec.BFDPort.NodeSelector, err) + klog.Error(err) + return portName, nil, err + } + if len(nodes) == 0 { + err = fmt.Errorf("no nodes found by selector %q", selector.String()) + klog.Error(err) + return portName, nil, err + } + + nodeNames := make([]string, 0, len(nodes)) + chassisCount = min(chassisCount, len(nodes)) + chassisNames := make([]string, 0, chassisCount) + for _, nodes := range nodes[:chassisCount] { + chassis, err := c.OVNSbClient.GetChassisByHost(nodes.Name) + if err != nil { + err = fmt.Errorf("failed to get chassis of node %s: %w", nodes.Name, err) + klog.Error(err) + return portName, nil, err + } + chassisNames = append(chassisNames, chassis.Name) + nodeNames = append(nodeNames, nodes.Name) + } + + networks := strings.Split(vpc.Spec.BFDPort.IP, ",") + if err = c.OVNNbClient.CreateLogicalRouterPort(vpc.Name, portName, "", networks); err != nil { + klog.Error(err) + return portName, nil, err + } + if err = c.OVNNbClient.UpdateLogicalRouterPortNetworks(portName, networks); err != nil { + klog.Error(err) + return portName, nil, err + } + if err = c.OVNNbClient.UpdateLogicalRouterPortOptions(portName, map[string]string{"bfd-only": "true"}); err != nil { + klog.Error(err) + return portName, nil, err + } + if err = c.OVNNbClient.CreateHAChassisGroup(portName, chassisNames, map[string]string{"lrp": portName}); err != nil { + klog.Error(err) + return portName, nil, err + } + if err = c.OVNNbClient.SetLogicalRouterPortHAChassisGroup(portName, portName); err != nil { + klog.Error(err) + return portName, nil, err + } + + return portName, nodeNames, nil +} + func (c *Controller) addPolicyRouteToVpc(vpcName string, policy *kubeovnv1.PolicyRoute, externalIDs map[string]string) error { var ( nextHops []string @@ -664,7 +766,7 @@ func (c *Controller) addPolicyRouteToVpc(vpcName string, policy *kubeovnv1.Polic nextHops = strings.Split(policy.NextHopIP, ",") } - if err = c.OVNNbClient.AddLogicalRouterPolicy(vpcName, policy.Priority, policy.Match, string(policy.Action), nextHops, externalIDs); err != nil { + if err = c.OVNNbClient.AddLogicalRouterPolicy(vpcName, policy.Priority, policy.Match, string(policy.Action), nextHops, nil, externalIDs); err != nil { klog.Errorf("add policy route to vpc %s failed, %v", vpcName, err) return err } @@ -856,19 +958,19 @@ func (c *Controller) formatVpc(vpc *kubeovnv1.Vpc) (*kubeovnv1.Vpc, error) { changed = true } if item.Policy != kubeovnv1.PolicyDst && item.Policy != kubeovnv1.PolicySrc { - return nil, fmt.Errorf("unknown policy type: %s", item.Policy) + return nil, fmt.Errorf("unknown policy type: %q", item.Policy) } // check cidr if strings.Contains(item.CIDR, "/") { if _, _, err := net.ParseCIDR(item.CIDR); err != nil { - return nil, fmt.Errorf("invalid cidr %s: %w", item.CIDR, err) + return nil, fmt.Errorf("invalid cidr %q: %w", item.CIDR, err) } } else if ip := net.ParseIP(item.CIDR); ip == nil { - return nil, fmt.Errorf("invalid ip %s", item.CIDR) + return nil, fmt.Errorf("invalid ip %q", item.CIDR) } // check next hop ip if ip := net.ParseIP(item.NextHopIP); ip == nil { - return nil, fmt.Errorf("invalid next hop ip %s", item.NextHopIP) + return nil, fmt.Errorf("invalid next hop ip %q", item.NextHopIP) } } @@ -896,7 +998,7 @@ func (c *Controller) formatVpc(vpc *kubeovnv1.Vpc) (*kubeovnv1.Vpc, error) { klog.Errorf("failed to update vpc %s: %v", vpc.Name, err) return nil, err } - return newVpc, err + return newVpc, nil } return vpc, nil @@ -1159,7 +1261,7 @@ func (c *Controller) handleDelVpcExternalSubnet(key, subnet string) error { return err } } - if err := c.OVNNbClient.DeleteBFD(lrpName, ""); err != nil { + if err := c.OVNNbClient.DeleteBFDByDstIP(lrpName, ""); err != nil { klog.Error(err) return err } diff --git a/pkg/controller/vpc_egress_gateway.go b/pkg/controller/vpc_egress_gateway.go new file mode 100644 index 00000000000..5607d0e1929 --- /dev/null +++ b/pkg/controller/vpc_egress_gateway.go @@ -0,0 +1,819 @@ +package controller + +import ( + "context" + "encoding/json" + "fmt" + "maps" + "net/netip" + "reflect" + "slices" + "strconv" + "strings" + + nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + "k8s.io/utils/set" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/ovs" + "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" + "github.com/kubeovn/kube-ovn/pkg/request" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +func (c *Controller) enqueueAddVpcEgressGateway(obj interface{}) { + key, err := cache.MetaNamespaceKeyFunc(obj) + if err != nil { + utilruntime.HandleError(err) + return + } + klog.V(3).Infof("enqueue add vpc-egress-gateway %s", key) + c.addOrUpdateVpcEgressGatewayQueue.Add(key) +} + +func (c *Controller) enqueueUpdateVpcEgressGateway(_, newObj interface{}) { + key, err := cache.MetaNamespaceKeyFunc(newObj) + if err != nil { + utilruntime.HandleError(err) + return + } + klog.V(3).Infof("enqueue update vpc-egress-gateway %s", key) + c.addOrUpdateVpcEgressGatewayQueue.Add(key) +} + +func (c *Controller) enqueueDeleteVpcEgressGateway(obj interface{}) { + key, err := cache.MetaNamespaceKeyFunc(obj) + if err != nil { + utilruntime.HandleError(err) + return + } + klog.V(3).Infof("enqueue delete vpc-egress-gateway %s", key) + c.delVpcEgressGatewayQueue.Add(key) +} + +func (c *Controller) handleAddOrUpdateVpcEgressGateway(key string) error { + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + c.vpcEgressGatewayKeyMutex.LockKey(key) + defer func() { _ = c.vpcEgressGatewayKeyMutex.UnlockKey(key) }() + + cachedGateway, err := c.vpcEgressGatewayLister.VpcEgressGateways(ns).Get(name) + if err != nil { + if !k8serrors.IsNotFound(err) { + klog.Error(err) + return err + } + return nil + } + + if !cachedGateway.DeletionTimestamp.IsZero() { + c.delVpcEgressGatewayQueue.Add(key) + return nil + } + + klog.Infof("reconciling vpc-egress-gateway %s", key) + gw := cachedGateway.DeepCopy() + if gw, err = c.initVpcEgressGatewayStatus(gw); err != nil { + return err + } + + vpcName := gw.Spec.VPC + if vpcName == "" { + vpcName = c.config.ClusterRouter + } + vpc, err := c.vpcsLister.Get(vpcName) + if err != nil { + klog.Error(err) + return err + } + if gw.Spec.BFD.Enabled && !vpc.Status.BFDPort.Enabled { + err = fmt.Errorf("vpc %s bfd port is not enabled or not ready", vpc.Name) + klog.Error(err) + gw.Status.Conditions.SetCondition(kubeovnv1.Validated, corev1.ConditionFalse, "VpcBfdPortNotEnabled", err.Error(), gw.Generation) + _, _ = c.updateVpcEgressGatewayStatus(gw) + return err + } + + if controllerutil.AddFinalizer(gw, util.KubeOVNControllerFinalizer) { + gw.Spec.VPC = vpcName + if gw, err = c.config.KubeOvnClient.KubeovnV1().VpcEgressGateways(gw.Namespace). + Update(context.Background(), gw, metav1.UpdateOptions{}); err != nil { + err = fmt.Errorf("failed to add finalizer for vpc-egress-gateway %s: %w", gw.Name, err) + klog.Error(err) + return err + } + } + + var bfdIP, bfdIPv4, bfdIPv6 string + if gw.Spec.BFD.Enabled { + bfdIP = vpc.Status.BFDPort.IP + bfdIPv4, bfdIPv6 = util.SplitStringIP(bfdIP) + } + + ipv4Src, ipv6Src, deploy, err := c.reconcileVpcEgressGatewayWorkload(gw, vpc, bfdIP, bfdIPv4, bfdIPv6) + if err != nil { + klog.Error(err) + gw.Status.Conditions.SetCondition(kubeovnv1.Ready, corev1.ConditionFalse, "ReconcileWorkloadFailed", err.Error(), gw.Generation) + _, _ = c.updateVpcEgressGatewayStatus(gw) + return err + } + + var podIPs []string + gw.Status.InternalIPs = nil + gw.Status.ExternalIPs = nil + gw.Status.Workload = deploy.Name + gw.Status.Ready = util.DeploymentIsReady(deploy) + if !gw.Status.Ready { + msg := fmt.Sprintf("Waiting for deployment %s to be ready", deploy.Name) + gw.Status.Conditions.SetCondition(kubeovnv1.Ready, corev1.ConditionFalse, "Processing", msg, gw.Generation) + } else { + podSelector, err := metav1.LabelSelectorAsSelector(deploy.Spec.Selector) + if err != nil { + err = fmt.Errorf("failed to get pod selector of deployment %s: %w", deploy.Name, err) + klog.Error(err) + return err + } + + pods, err := c.podsLister.Pods(deploy.Namespace).List(podSelector) + if err != nil { + err = fmt.Errorf("failed to list pods of deployment %s: %w", deploy.Name, err) + klog.Error(err) + return err + } + + for _, pod := range pods { + ips := util.PodIPs(*pod) + podIPs = append(podIPs, ips...) + gw.Status.InternalIPs = append(gw.Status.InternalIPs, strings.Join(ips, ",")) + if pod.Annotations[nadv1.NetworkStatusAnnot] == "" { + klog.Errorf("pod %s/%s has no network status annotation", pod.Namespace, pod.Name) + continue + } + var status []nadv1.NetworkStatus + if err = json.Unmarshal([]byte(pod.Annotations[nadv1.NetworkStatusAnnot]), &status); err != nil { + klog.Errorf("failed to unmarshal network status annotation of pod %s/%s: %v", pod.Namespace, pod.Name, err) + continue + } + for _, s := range status { + if !s.Default { + gw.Status.ExternalIPs = append(gw.Status.ExternalIPs, strings.Join(s.IPs, ",")) + break + } + } + } + } + if gw, err = c.updateVpcEgressGatewayStatus(gw); err != nil { + klog.Error(err) + return err + } + if !gw.Status.Ready { + return nil + } + + nextHopsIPv4, hextHopsIPv6 := util.SplitIpsByProtocol(podIPs) + if err = c.reconcileVpcEgressGatewayOVNRoutes(gw, 4, vpc.Status.Router, vpc.Status.BFDPort.Name, bfdIPv4, set.New(nextHopsIPv4...), ipv4Src); err != nil { + klog.Error(err) + return err + } + if err = c.reconcileVpcEgressGatewayOVNRoutes(gw, 6, vpc.Status.Router, vpc.Status.BFDPort.Name, bfdIPv6, set.New(hextHopsIPv6...), ipv6Src); err != nil { + klog.Error(err) + return err + } + + gw.Status.Ready = true + gw.Status.Phase = kubeovnv1.PhaseCompleted + gw.Status.Conditions.SetReady("ReconcileSuccess", gw.Generation) + if _, err = c.updateVpcEgressGatewayStatus(gw); err != nil { + return err + } + + return nil +} + +func (c *Controller) initVpcEgressGatewayStatus(gw *kubeovnv1.VpcEgressGateway) (*kubeovnv1.VpcEgressGateway, error) { + var err error + if gw.Status.Phase == "" || gw.Status.Phase == kubeovnv1.PhasePending { + gw.Status.Phase = kubeovnv1.PhaseProcessing + gw, err = c.updateVpcEgressGatewayStatus(gw) + } + return gw, err +} + +func (c *Controller) updateVpcEgressGatewayStatus(gw *kubeovnv1.VpcEgressGateway) (*kubeovnv1.VpcEgressGateway, error) { + if len(gw.Status.Conditions) == 0 { + gw.Status.Conditions.SetCondition(kubeovnv1.Init, corev1.ConditionUnknown, "Processing", "", gw.Generation) + } + if !gw.Status.Ready { + gw.Status.Phase = kubeovnv1.PhaseProcessing + } + + var err error + if gw, err = c.config.KubeOvnClient.KubeovnV1().VpcEgressGateways(gw.Namespace). + UpdateStatus(context.Background(), gw, metav1.UpdateOptions{}); err != nil { + err = fmt.Errorf("failed to update status of vpc-egress-gateway %s: %w", gw.Name, err) + klog.Error(err) + return nil, err + } + + return gw, nil +} + +func (c *Controller) reconcileVpcEgressGatewayWorkload(gw *kubeovnv1.VpcEgressGateway, vpc *kubeovnv1.Vpc, bfdIP, bfdIPv4, bfdIPv6 string) (set.Set[string], set.Set[string], *appsv1.Deployment, error) { + image := c.config.Image + if gw.Spec.Image != "" { + image = gw.Spec.Image + } + if image == "" { + err := fmt.Errorf("no image specified for vpc egress gateway %s", gw.Name) + klog.Error(err) + return nil, nil, nil, err + } + + if len(gw.Spec.InternalIPs) != 0 && len(gw.Spec.InternalIPs) < int(gw.Spec.Replicas) { + err := fmt.Errorf("internal IPs count %d is less than replicas %d", len(gw.Spec.InternalIPs), gw.Spec.Replicas) + klog.Error(err) + return nil, nil, nil, err + } + if len(gw.Spec.ExternalIPs) != 0 && len(gw.Spec.ExternalIPs) < int(gw.Spec.Replicas) { + err := fmt.Errorf("external IPs count %d is less than replicas %d", len(gw.Spec.ExternalIPs), gw.Spec.Replicas) + klog.Error(err) + return nil, nil, nil, err + } + + internalSubnet := gw.Spec.InternalSubnet + if internalSubnet == "" { + internalSubnet = vpc.Status.DefaultLogicalSwitch + } + if internalSubnet == "" { + err := fmt.Errorf("default subnet of vpc %s not found, please set internal subnet of the egress gateway", vpc.Name) + klog.Error(err) + return nil, nil, nil, err + } + intSubnet, err := c.subnetsLister.Get(internalSubnet) + if err != nil { + klog.Error(err) + return nil, nil, nil, err + } + extSubnet, err := c.subnetsLister.Get(gw.Spec.ExternalSubnet) + if err != nil { + klog.Error(err) + return nil, nil, nil, err + } + if !strings.ContainsRune(extSubnet.Spec.Provider, '.') { + err = fmt.Errorf("please set correct provider of subnet %s to get the network-attachment-definition", extSubnet.Name) + klog.Error(err) + return nil, nil, nil, err + } + subStrings := strings.Split(extSubnet.Spec.Provider, ".") + nadName, nadNamespace := subStrings[0], subStrings[1] + if _, err = c.config.AttachNetClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(nadNamespace). + Get(context.Background(), nadName, metav1.GetOptions{}); err != nil { + klog.Errorf("failed to get net-attach-def %s/%s: %v", nadNamespace, nadName, err) + return nil, nil, nil, err + } + + routes := make(ExtraPodRoutes) + intCIDRIPv4, intCIDRIPv6 := util.SplitStringIP(intSubnet.Spec.CIDRBlock) + intGatewayIPv4, intGatewayIPv6 := util.SplitStringIP(intSubnet.Spec.Gateway) + extGatewayIPv4, extGatewayIPv6 := util.SplitStringIP(extSubnet.Spec.Gateway) + routes.Add(util.OvnProvider, bfdIPv4, intGatewayIPv4) + routes.Add(util.OvnProvider, bfdIPv6, intGatewayIPv6) + routes.Add(extSubnet.Spec.Provider, "0.0.0.0/0", extGatewayIPv4) + routes.Add(extSubnet.Spec.Provider, "::/0", extGatewayIPv6) + + annotations, err := routes.ToAnnotations() + if err != nil { + klog.Error(err) + return nil, nil, nil, err + } + annotations[util.AttachmentNetworkAnnotation] = fmt.Sprintf("%s/%s", nadNamespace, nadName) + annotations[util.LogicalSwitchAnnotation] = intSubnet.Name + if len(gw.Spec.InternalIPs) != 0 { + annotations[util.IPPoolAnnotation] = strings.Join(gw.Spec.InternalIPs, ";") + } + if len(gw.Spec.ExternalIPs) != 0 { + annotations[fmt.Sprintf(util.IPPoolAnnotationTemplate, extSubnet.Spec.Provider)] = strings.Join(gw.Spec.ExternalIPs, ";") + } + + ipv4ForwardSrc, ipv6ForwardSrc := set.New[string](), set.New[string]() + ipv4SNATSrc, ipv6SNATSrc := set.New[string](), set.New[string]() + for _, policy := range gw.Spec.Policies { + ipv4, ipv6 := util.SplitIpsByProtocol(policy.IPBlocks) + if policy.SNAT { + ipv4SNATSrc.Insert(ipv4...) + ipv6SNATSrc.Insert(ipv6...) + } else { + ipv4ForwardSrc.Insert(ipv4...) + ipv6ForwardSrc.Insert(ipv6...) + } + for _, subnetName := range policy.Subnets { + subnet, err := c.subnetsLister.Get(subnetName) + if err != nil { + klog.Error(err) + return nil, nil, nil, err + } + if subnet.Status.IsNotValidated() { + err = fmt.Errorf("subnet %s is not validated", subnet.Name) + klog.Error(err) + return nil, nil, nil, err + } + // TODO: check subnet's vpc and vlan + ipv4, ipv6 := util.SplitStringIP(subnet.Spec.CIDRBlock) + if policy.SNAT { + ipv4SNATSrc.Insert(ipv4) + ipv6SNATSrc.Insert(ipv6) + } else { + ipv4ForwardSrc.Insert(ipv4) + ipv6ForwardSrc.Insert(ipv6) + } + } + } + + intRouteDstIPv4, intRouteDstIPv6 := ipv4ForwardSrc.Union(ipv4SNATSrc), ipv6ForwardSrc.Union(ipv6SNATSrc) + intRouteDstIPv4.Insert(bfdIPv4) + intRouteDstIPv6.Insert(bfdIPv6) + intRouteDstIPv4.Delete("") + intRouteDstIPv6.Delete("") + ipv4SNATSrc.Delete("") + ipv6SNATSrc.Delete("") + + initEnv, err := vpcEgressGatewayInitContainerEnv(4, intGatewayIPv4, intCIDRIPv4, intRouteDstIPv4, ipv4SNATSrc) + if err != nil { + klog.Error(err) + return nil, nil, nil, err + } + ipv6Env, err := vpcEgressGatewayInitContainerEnv(6, intGatewayIPv6, intCIDRIPv6, intRouteDstIPv6, ipv6SNATSrc) + if err != nil { + klog.Error(err) + return nil, nil, nil, err + } + initEnv = append(initEnv, ipv6Env...) + + // generate workload + labels := map[string]string{util.VpcEgressGatewayAnnotation: gw.Name} + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("vpc-egress-gateway-%s", gw.Name), + Namespace: gw.Namespace, + Labels: labels, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxUnavailable: ptr.To(intstr.FromInt(1)), + MaxSurge: ptr.To(intstr.FromInt(1)), + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Annotations: annotations, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: mergeNodeSelector(gw.Spec.NodeSelector), + }, + PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + TopologyKey: "kubernetes.io/hostname", + }}, + }, + }, + InitContainers: []corev1.Container{{ + Name: "init", + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"bash", "/kube-ovn/init-vpc-egress-gateway.sh"}, + Env: initEnv, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + }, + }}, + Containers: []corev1.Container{{ + Name: "gateway", + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"sleep", "infinity"}, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(false), + RunAsUser: ptr.To[int64](65534), + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"NET_ADMIN", "NET_RAW"}, + Drop: []corev1.Capability{"ALL"}, + }, + }, + }}, + TerminationGracePeriodSeconds: ptr.To[int64](0), + }, + }, + }, + } + if err = util.SetOwnerReference(gw, deploy); err != nil { + klog.Error(err) + return nil, nil, nil, err + } + + if bfdIP != "" { + container := vpcEgressGatewayContainerBFDD(image, bfdIP, gw.Spec.BFD.MinTX, gw.Spec.BFD.MinRX, gw.Spec.BFD.Multiplier) + deploy.Spec.Template.Spec.Containers[0] = container + } + + hash, err := util.Sha256HashObject(deploy) + if err != nil { + err = fmt.Errorf("failed to hash generated deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return nil, nil, nil, err + } + + hash = hash[:12] + deploy.Spec.Replicas = ptr.To(gw.Spec.Replicas) + deploy.Annotations = map[string]string{"ovn.kubernetes.io/generate-hash": hash} + if currentDeploy, err := c.deploymentsLister.Deployments(gw.Namespace).Get(deploy.Name); err != nil { + if !k8serrors.IsNotFound(err) { + err = fmt.Errorf("failed to get deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return nil, nil, nil, err + } + if deploy, err = c.config.KubeClient.AppsV1().Deployments(gw.Namespace). + Create(context.Background(), deploy, metav1.CreateOptions{}); err != nil { + err = fmt.Errorf("failed to create deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return nil, nil, nil, err + } + } else if !reflect.DeepEqual(currentDeploy.Spec.Replicas, deploy.Spec.Replicas) || + currentDeploy.Annotations["ovn.kubernetes.io/generate-hash"] != hash { + if deploy, err = c.config.KubeClient.AppsV1().Deployments(gw.Namespace). + Update(context.Background(), deploy, metav1.UpdateOptions{}); err != nil { + err = fmt.Errorf("failed to update deployment %s/%s: %w", deploy.Namespace, deploy.Name, err) + klog.Error(err) + return nil, nil, nil, err + } + } else { + deploy = currentDeploy + } + + intRouteDstIPv4.Delete(bfdIPv4) + intRouteDstIPv6.Delete(bfdIPv6) + return intRouteDstIPv4, intRouteDstIPv6, deploy, nil +} + +func (c *Controller) reconcileVpcEgressGatewayOVNRoutes(gw *kubeovnv1.VpcEgressGateway, af int, lrName, lrpName, bfdIP string, nextHops, sources set.Set[string]) error { + externalIDs := map[string]string{ + ovs.ExternalIDVendor: util.CniTypeName, + ovs.ExternalIDVpcEgressGateway: fmt.Sprintf("%s/%s", gw.Namespace, gw.Name), + "af": strconv.Itoa(af), + } + bfdList, err := c.OVNNbClient.FindBFD(externalIDs) + if err != nil { + klog.Error(err) + return err + } + + bfdIDs := set.New[string]() + bfdDstIPs := nextHops.Clone() + bfdMap := make(map[string]*string, nextHops.Len()) + for _, bfd := range bfdList { + if bfdIP == "" || bfd.LogicalPort != lrpName || !bfdDstIPs.Has(bfd.DstIP) { + if err = c.OVNNbClient.DeleteBFD(bfd.UUID); err != nil { + err = fmt.Errorf("failed to delete bfd %s: %w", bfd.UUID, err) + klog.Error(err) + return err + } + } + if bfdIP == "" || bfd.LogicalPort == lrpName && bfdDstIPs.Has(bfd.DstIP) { + // TODO: update min_rx, min_tx and multiplier + if bfdIP != "" { + bfdIDs.Insert(bfd.UUID) + bfdMap[bfd.DstIP] = ptr.To(bfd.UUID) + } + bfdDstIPs.Delete(bfd.DstIP) + } + } + if bfdIP != "" { + for _, dstIP := range bfdDstIPs.UnsortedList() { + bfd, err := c.OVNNbClient.CreateBFD(lrpName, dstIP, gw.Spec.BFD.MinRX, gw.Spec.BFD.MinTX, gw.Spec.BFD.Multiplier, externalIDs) + if err != nil { + klog.Error(err) + return err + } + bfdIDs.Insert(bfd.UUID) + bfdMap[bfd.DstIP] = ptr.To(bfd.UUID) + } + } + + // if lrName == c.config.ClusterRouter { + // reconcile LR policy + policies, err := c.OVNNbClient.ListLogicalRouterPolicies(lrName, util.EgressGatewayPolicyPriority, externalIDs, false) + if err != nil { + klog.Error(err) + return err + } + matches := set.New[string]() + for _, src := range sources.UnsortedList() { + matches.Insert(fmt.Sprintf("ip%d.src == %s", af, src)) + } + for _, policy := range policies { + if matches.Has(policy.Match) { + if !nextHops.Equal(set.New(policy.Nexthops...)) || !bfdIDs.Equal(set.New(policy.BFDSessions...)) { + policy.Nexthops, policy.BFDSessions = nextHops.UnsortedList(), bfdIDs.UnsortedList() + if err = c.OVNNbClient.UpdateLogicalRouterPolicy(policy, &policy.Nexthops, &policy.BFDSessions); err != nil { + err = fmt.Errorf("failed to update bfd sessions of logical router policy %s: %w", policy.UUID, err) + klog.Error(err) + return err + } + } + matches.Delete(policy.Match) + continue + } + if err = c.OVNNbClient.DeleteLogicalRouterPolicyByUUID(lrName, policy.UUID); err != nil { + err = fmt.Errorf("failed to delete ovn lr policy %q: %w", policy.Match, err) + klog.Error(err) + return err + } + } + bfdSessions := bfdIDs.UnsortedList() + for _, match := range matches.UnsortedList() { + if err := c.OVNNbClient.AddLogicalRouterPolicy(lrName, util.EgressGatewayPolicyPriority, match, + ovnnb.LogicalRouterPolicyActionReroute, nextHops.UnsortedList(), bfdSessions, externalIDs); err != nil { + klog.Error(err) + return err + } + } + + return nil +} + +func mergeNodeSelector(nodeSelector []kubeovnv1.VpcEgressGatewayNodeSelector) *corev1.NodeSelector { + if len(nodeSelector) == 0 { + return nil + } + + result := &corev1.NodeSelector{ + NodeSelectorTerms: make([]corev1.NodeSelectorTerm, len(nodeSelector)), + } + for i, selector := range nodeSelector { + result.NodeSelectorTerms[i] = corev1.NodeSelectorTerm{ + MatchExpressions: make([]corev1.NodeSelectorRequirement, len(selector.MatchExpressions), len(selector.MatchLabels)+len(selector.MatchExpressions)), + MatchFields: make([]corev1.NodeSelectorRequirement, len(selector.MatchFields)), + } + for j := range selector.MatchExpressions { + selector.MatchExpressions[j].DeepCopyInto(&result.NodeSelectorTerms[i].MatchExpressions[j]) + } + for _, key := range slices.Sorted(maps.Keys(selector.MatchLabels)) { + result.NodeSelectorTerms[i].MatchExpressions = append(result.NodeSelectorTerms[i].MatchExpressions, corev1.NodeSelectorRequirement{ + Key: key, + Operator: corev1.NodeSelectorOpIn, + Values: []string{selector.MatchLabels[key]}, + }) + } + for j := range selector.MatchFields { + selector.MatchFields[j].DeepCopyInto(&result.NodeSelectorTerms[i].MatchFields[j]) + } + } + + return result +} + +func vpcEgressGatewayInitContainerEnv(af int, internalGateway, internalCIDR string, routeDst, snatSrc set.Set[string]) ([]corev1.EnvVar, error) { + if internalGateway == "" { + return nil, nil + } + + cidr, err := netip.ParsePrefix(internalCIDR) + if err != nil { + err = fmt.Errorf("failed to parse internal CIDR %s: %w", internalCIDR, err) + klog.Error(err) + return nil, err + } + + dstList := make([]string, 0, routeDst.Len()) + for _, dst := range routeDst.SortedList() { + if !strings.ContainsRune(dst, '/') { + if af == 4 { + dst += "/32" + } else { + dst += "/128" + } + } + prefix, err := netip.ParsePrefix(dst) + if err != nil { + err = fmt.Errorf("failed to parse internal route destination %s: %w", dst, err) + klog.Error(err) + return nil, err + } + if !cidr.Overlaps(prefix) { + dstList = append(dstList, prefix.Masked().String()) + } + } + + return []corev1.EnvVar{{ + Name: fmt.Sprintf("INTERNAL_GATEWAY_IPV%d", af), + Value: internalGateway, + }, { + Name: fmt.Sprintf("INTERNAL_ROUTE_DST_IPV%d", af), + Value: strings.Join(dstList, ","), + }, { + Name: fmt.Sprintf("SNAT_SOURCES_IPV%d", af), + Value: strings.Join(snatSrc.SortedList(), ","), + }}, nil +} + +func vpcEgressGatewayContainerBFDD(image, bfdIP string, minTX, minRX, multiplier int) corev1.Container { + return corev1.Container{ + Name: "bfdd", + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"bash", "/kube-ovn/start-bfdd.sh"}, + Env: []corev1.EnvVar{{ + Name: "POD_IPS", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIPs", + }, + }, + }, { + Name: "BFD_PEER_IPS", + Value: bfdIP, + }, { + Name: "BFD_MIN_TX", + Value: strconv.Itoa(minTX), + }, { + Name: "BFD_MIN_RX", + Value: strconv.Itoa(minRX), + }, { + Name: "BFD_MULTI", + Value: strconv.Itoa(multiplier), + }}, + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"bash", "/kube-ovn/bfdd-prestart.sh"}, + }, + }, + InitialDelaySeconds: 1, + FailureThreshold: 1, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"bfdd-control", "status"}, + }, + }, + InitialDelaySeconds: 1, + PeriodSeconds: 5, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"bfdd-control", "status"}, + }, + }, + InitialDelaySeconds: 3, + PeriodSeconds: 3, + FailureThreshold: 1, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(false), + RunAsUser: ptr.To[int64](65534), + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"NET_ADMIN", "NET_BIND_SERVICE", "NET_RAW"}, + Drop: []corev1.Capability{"ALL"}, + }, + }, + } +} + +func (c *Controller) handleDelVpcEgressGateway(key string) error { + ns, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + c.vpcEgressGatewayKeyMutex.LockKey(key) + defer func() { _ = c.vpcEgressGatewayKeyMutex.UnlockKey(key) }() + + cachedGateway, err := c.vpcEgressGatewayLister.VpcEgressGateways(ns).Get(name) + if err != nil { + if !k8serrors.IsNotFound(err) { + err = fmt.Errorf("failed to get vpc-egress-gateway %s: %w", key, err) + klog.Error(err) + return err + } + return nil + } + + klog.Infof("handle deleting vpc-egress-gateway %s", key) + if err = c.cleanOVNforVpcEgressGateway(key, cachedGateway.Spec.VPC); err != nil { + klog.Error(err) + return err + } + + gw := cachedGateway.DeepCopy() + if controllerutil.RemoveFinalizer(gw, util.KubeOVNControllerFinalizer) { + if _, err = c.config.KubeOvnClient.KubeovnV1().VpcEgressGateways(gw.Namespace). + Update(context.Background(), gw, metav1.UpdateOptions{}); err != nil { + err = fmt.Errorf("failed to remove finalizer from vpc-egress-gateway %s: %w", key, err) + klog.Error(err) + } + } + + return nil +} + +func (c *Controller) cleanOVNforVpcEgressGateway(key, lrName string) error { + externalIDs := map[string]string{ + ovs.ExternalIDVendor: util.CniTypeName, + ovs.ExternalIDVpcEgressGateway: key, + } + + bfdList, err := c.OVNNbClient.FindBFD(externalIDs) + if err != nil { + klog.Error(err) + return err + } + for _, bfd := range bfdList { + if err = c.OVNNbClient.DeleteBFD(bfd.UUID); err != nil { + klog.Error(err) + return err + } + } + + if lrName == "" { + lrName = c.config.ClusterRouter + } + if err = c.OVNNbClient.DeleteLogicalRouterPolicies(lrName, -1, externalIDs); err != nil { + klog.Error(err) + return err + } + if err = c.OVNNbClient.DeleteLogicalRouterStaticRouteByExternalIDs(lrName, externalIDs); err != nil { + klog.Error(err) + return err + } + + return nil +} + +type ExtraPodRoutes map[string]map[string][]string + +func (r ExtraPodRoutes) Add(provider, destination, gateway string) { + if gateway == "" || destination == "" { + return + } + + if _, ok := r[provider]; !ok { + r[provider] = make(map[string][]string) + } + r[provider][gateway] = append(r[provider][gateway], destination) +} + +func (r ExtraPodRoutes) ToAnnotations() (map[string]string, error) { + annotations := make(map[string]string, len(r)) + for provider, routesMap := range r { + var routes []request.Route + for gw := range routesMap { + if gw == "" { + continue + } + for _, dst := range routesMap[gw] { + routes = append(routes, request.Route{ + Destination: dst, + Gateway: gw, + }) + } + } + if len(routes) == 0 { + continue + } + + buf, err := json.Marshal(routes) + if err != nil { + klog.Error(err) + return nil, err + } + annotations[fmt.Sprintf(util.RoutesAnnotationTemplate, provider)] = string(buf) + } + return annotations, nil +} diff --git a/pkg/ovn_ic_controller/ovn_ic_controller.go b/pkg/ovn_ic_controller/ovn_ic_controller.go index 3757862ef77..d83784e7224 100644 --- a/pkg/ovn_ic_controller/ovn_ic_controller.go +++ b/pkg/ovn_ic_controller/ovn_ic_controller.go @@ -612,7 +612,7 @@ func (c *Controller) syncOneRouteToPolicy(key, value string) { match = util.MatchV6Dst + " == " + prefix } - if err = c.OVNNbClient.AddLogicalRouterPolicy(lr.Name, util.OvnICPolicyPriority, match, ovnnb.LogicalRouterPolicyActionAllow, nil, map[string]string{key: value, "vendor": util.CniTypeName}); err != nil { + if err = c.OVNNbClient.AddLogicalRouterPolicy(lr.Name, util.OvnICPolicyPriority, match, ovnnb.LogicalRouterPolicyActionAllow, nil, nil, map[string]string{key: value, "vendor": util.CniTypeName}); err != nil { klog.Errorf("failed to add router policy: %v", err) } diff --git a/pkg/ovs/interface.go b/pkg/ovs/interface.go index df6907c4d29..67a230b9a16 100644 --- a/pkg/ovs/interface.go +++ b/pkg/ovs/interface.go @@ -39,7 +39,9 @@ type LogicalRouterPort interface { CreatePeerRouterPort(localRouter, remoteRouter, localRouterPortIP string) error CreateLogicalRouterPort(lrName, lrpName, mac string, networks []string) error UpdateLogicalRouterPortRA(lrpName, ipv6RAConfigsStr string, enableIPv6RA bool) error + UpdateLogicalRouterPortNetworks(lrpName string, networks []string) error UpdateLogicalRouterPortOptions(lrpName string, options map[string]string) error + SetLogicalRouterPortHAChassisGroup(lrpName, haChassisGroupName string) error DeleteLogicalRouterPort(lrpName string) error DeleteLogicalRouterPorts(externalIDs map[string]string, filter func(lrp *ovnnb.LogicalRouterPort) bool) error GetLogicalRouterPort(lrpName string, ignoreNotFound bool) (*ovnnb.LogicalRouterPort, error) @@ -49,16 +51,24 @@ type LogicalRouterPort interface { LogicalRouterPortExists(lrpName string) (bool, error) } +type HAChassisGroup interface { + CreateHAChassisGroup(name string, chassises []string, externalIDs map[string]string) error + GetHAChassisGroup(name string, ignoreNotFound bool) (*ovnnb.HAChassisGroup, error) + DeleteHAChassisGroup(name string) error +} + type GatewayChassis interface { UpdateGatewayChassis(gwChassis *ovnnb.GatewayChassis, fields ...interface{}) error } type BFD interface { - CreateBFD(lrpName, dstIP string, minRx, minTx, detectMult int) (*ovnnb.BFD, error) - DeleteBFD(lrpName, dstIP string) error + CreateBFD(lrpName, dstIP string, minRx, minTx, detectMult int, externalIDs map[string]string) (*ovnnb.BFD, error) + DeleteBFD(uuid string) error + DeleteBFDByDstIP(lrpName, dstIP string) error ListBFDs(lrpName, dstIP string) ([]ovnnb.BFD, error) ListDownBFDs(dstIP string) ([]ovnnb.BFD, error) ListUpBFDs(dstIP string) ([]ovnnb.BFD, error) + FindBFD(externalIDs map[string]string) ([]ovnnb.BFD, error) UpdateBFD(bfd *ovnnb.BFD, fields ...interface{}) error MonitorBFD() } @@ -167,15 +177,18 @@ type AddressSet interface { type LogicalRouterStaticRoute interface { AddLogicalRouterStaticRoute(lrName, routeTable, policy, ipPrefix string, bfdID *string, nexthops ...string) error + UpdateLogicalRouterStaticRoute(route *ovnnb.LogicalRouterStaticRoute, fields ...interface{}) error ClearLogicalRouterStaticRoute(lrName string) error DeleteLogicalRouterStaticRoute(lrName string, routeTable, policy *string, ipPrefix, nextHop string) error + DeleteLogicalRouterStaticRouteByUUID(lrName, uuid string) error + DeleteLogicalRouterStaticRouteByExternalIDs(lrName string, externalIDs map[string]string) error ListLogicalRouterStaticRoutesByOption(lrName, routeTable, key, value string) ([]*ovnnb.LogicalRouterStaticRoute, error) ListLogicalRouterStaticRoutes(lrName string, routeTable, policy *string, ipPrefix string, externalIDs map[string]string) ([]*ovnnb.LogicalRouterStaticRoute, error) LogicalRouterStaticRouteExists(lrName, routeTable, policy, ipPrefix, nexthop string) (bool, error) } type LogicalRouterPolicy interface { - AddLogicalRouterPolicy(lrName string, priority int, match, action string, nextHops []string, externalIDs map[string]string) error + AddLogicalRouterPolicy(lrName string, priority int, match, action string, nextHops, bfdSessions []string, externalIDs map[string]string) error DeleteLogicalRouterPolicy(lrName string, priority int, match string) error DeleteLogicalRouterPolicies(lrName string, priority int, externalIDs map[string]string) error DeleteLogicalRouterPolicyByUUID(lrName, uuid string) error @@ -184,6 +197,7 @@ type LogicalRouterPolicy interface { ListLogicalRouterPolicies(lrName string, priority int, externalIDs map[string]string, ignoreExtIDEmptyValue bool) ([]*ovnnb.LogicalRouterPolicy, error) GetLogicalRouterPolicy(lrName string, priority int, match string, ignoreNotFound bool) ([]*ovnnb.LogicalRouterPolicy, error) GetLogicalRouterPoliciesByExtID(lrName, key, value string) ([]*ovnnb.LogicalRouterPolicy, error) + UpdateLogicalRouterPolicy(policy *ovnnb.LogicalRouterPolicy, fields ...interface{}) error } type NAT interface { @@ -210,6 +224,7 @@ type NbClient interface { BFD DHCPOptions GatewayChassis + HAChassisGroup LoadBalancer LoadBalancerHealthCheck LogicalRouterPolicy diff --git a/pkg/ovs/ovn-nb-bfd.go b/pkg/ovs/ovn-nb-bfd.go index 2ea3fa5d910..31e3396b1e5 100644 --- a/pkg/ovs/ovn-nb-bfd.go +++ b/pkg/ovs/ovn-nb-bfd.go @@ -5,13 +5,12 @@ import ( "fmt" "time" + "github.com/ovn-org/libovsdb/cache" + "github.com/ovn-org/libovsdb/model" "k8s.io/klog/v2" "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" "github.com/kubeovn/kube-ovn/pkg/util" - - "github.com/ovn-org/libovsdb/cache" - "github.com/ovn-org/libovsdb/model" ) func (c *OVNNbClient) ListBFDs(lrpName, dstIP string) ([]ovnnb.BFD, error) { @@ -68,7 +67,7 @@ func (c *OVNNbClient) ListUpBFDs(dstIP string) ([]ovnnb.BFD, error) { return bfdList, nil } -func (c *OVNNbClient) CreateBFD(lrpName, dstIP string, minRx, minTx, detectMult int) (*ovnnb.BFD, error) { +func (c *OVNNbClient) CreateBFD(lrpName, dstIP string, minRx, minTx, detectMult int, externalIDs map[string]string) (*ovnnb.BFD, error) { bfdList, err := c.ListBFDs(lrpName, dstIP) if err != nil { klog.Error(err) @@ -84,6 +83,7 @@ func (c *OVNNbClient) CreateBFD(lrpName, dstIP string, minRx, minTx, detectMult MinRx: &minRx, MinTx: &minTx, DetectMult: &detectMult, + ExternalIDs: externalIDs, } ops, err := c.Create(bfd) if err != nil { @@ -124,7 +124,22 @@ func (c *OVNNbClient) UpdateBFD(bfd *ovnnb.BFD, fields ...interface{}) error { return nil } -func (c *OVNNbClient) DeleteBFD(lrpName, dstIP string) error { +func (c *OVNNbClient) DeleteBFD(uuid string) error { + ops, err := c.Where(&ovnnb.BFD{UUID: uuid}).Delete() + if err != nil { + err := fmt.Errorf("failed to generate operations for BFD deletion with UUID %s: %w", uuid, err) + klog.Error(err) + return err + } + if err = c.Transact("bfd-del", ops); err != nil { + err = fmt.Errorf("failed to delete BFD with with UUID %s: %w", uuid, err) + klog.Error(err) + return err + } + return nil +} + +func (c *OVNNbClient) DeleteBFDByDstIP(lrpName, dstIP string) error { bfdList, err := c.ListBFDs(lrpName, dstIP) if err != nil { klog.Error(err) @@ -199,6 +214,10 @@ func (c *OVNNbClient) bfdAddL3HAHandler(table string, model model.Model) { } bfd := model.(*ovnnb.BFD) + if bfd.ExternalIDs[ExternalIDVpcEgressGateway] != "" { + return + } + klog.Infof("lrp %s add BFD to dst ip %s", bfd.LogicalPort, bfd.DstIP) needRecheck := false if bfd.Status == nil { @@ -230,6 +249,9 @@ func (c *OVNNbClient) bfdUpdateL3HAHandler(table string, oldModel, newModel mode oldBfd := oldModel.(*ovnnb.BFD) newBfd := newModel.(*ovnnb.BFD) + if newBfd.ExternalIDs[ExternalIDVpcEgressGateway] != "" { + return + } if oldBfd.Status == nil || newBfd.Status == nil { return } @@ -343,5 +365,32 @@ func (c *OVNNbClient) bfdDelL3HAHandler(table string, model model.Model) { return } bfd := model.(*ovnnb.BFD) + if bfd.ExternalIDs[ExternalIDVpcEgressGateway] != "" { + return + } klog.Infof("lrp %s del BFD to dst ip %s", bfd.LogicalPort, bfd.DstIP) } + +func (c *OVNNbClient) FindBFD(externalIDs map[string]string) ([]ovnnb.BFD, error) { + ctx, cancel := context.WithTimeout(context.Background(), c.Timeout) + defer cancel() + + bfdList := make([]ovnnb.BFD, 0) + if err := c.ovsDbClient.WhereCache(func(bfd *ovnnb.BFD) bool { + if len(bfd.ExternalIDs) == 0 && len(externalIDs) != 0 { + return false + } + for k, v := range externalIDs { + if bfd.ExternalIDs[k] != v { + return false + } + } + return true + }).List(ctx, &bfdList); err != nil { + err := fmt.Errorf("failed to find ovn BFD: %w", err) + klog.Error(err) + return nil, err + } + + return bfdList, nil +} diff --git a/pkg/ovs/ovn-nb-bfd_test.go b/pkg/ovs/ovn-nb-bfd_test.go index d63e34912a0..1642a1e0237 100644 --- a/pkg/ovs/ovn-nb-bfd_test.go +++ b/pkg/ovs/ovn-nb-bfd_test.go @@ -22,7 +22,7 @@ func (suite *OvnClientTestSuite) testCreateBFD() { lrpName := "test-create-bfd" - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) require.Equal(t, lrpName, bfd.LogicalPort) @@ -40,11 +40,11 @@ func (suite *OvnClientTestSuite) testCreateBFD() { lrpName := "test-create-existing-bfd" - bfd1, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd1, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd1) - bfd2, err := nbClient.CreateBFD(lrpName, dstIP, minRx+1, minTx+1, detectMult+1) + bfd2, err := nbClient.CreateBFD(lrpName, dstIP, minRx+1, minTx+1, detectMult+1, nil) require.NoError(t, err) require.NotNil(t, bfd2) require.Equal(t, bfd1, bfd2) @@ -71,11 +71,11 @@ func (suite *OvnClientTestSuite) testListBFD() { t.Run("list BFDs", func(t *testing.T) { t.Parallel() - bfd1, err := nbClient.CreateBFD(lrpName, dstIP1, minRx1, minTx1, detectMult1) + bfd1, err := nbClient.CreateBFD(lrpName, dstIP1, minRx1, minTx1, detectMult1, nil) require.NoError(t, err) require.NotNil(t, bfd1) - bfd2, err := nbClient.CreateBFD(lrpName, dstIP2, minRx2, minTx2, detectMult2) + bfd2, err := nbClient.CreateBFD(lrpName, dstIP2, minRx2, minTx2, detectMult2, nil) require.NoError(t, err) require.NotNil(t, bfd2) @@ -101,7 +101,7 @@ func (suite *OvnClientTestSuite) testListBFD() { t.Run("closed server list failed BFDs", func(t *testing.T) { t.Parallel() - failedBFD1, err := failedNbClient.CreateBFD(lrpName, dstIP1, minRx1, minTx1, detectMult1) + failedBFD1, err := failedNbClient.CreateBFD(lrpName, dstIP1, minRx1, minTx1, detectMult1, nil) require.Error(t, err) require.Nil(t, failedBFD1) // cache db should be empty @@ -123,14 +123,14 @@ func (suite *OvnClientTestSuite) testDeleteBFD() { minRx1, minTx1, detectMult1 := 101, 102, 19 minRx2, minTx2, detectMult2 := 201, 202, 29 - _, err := nbClient.CreateBFD(lrpName, dstIP1, minRx1, minTx1, detectMult1) + _, err := nbClient.CreateBFD(lrpName, dstIP1, minRx1, minTx1, detectMult1, nil) require.NoError(t, err) - bfd2, err := nbClient.CreateBFD(lrpName, dstIP2, minRx2, minTx2, detectMult2) + bfd2, err := nbClient.CreateBFD(lrpName, dstIP2, minRx2, minTx2, detectMult2, nil) require.NoError(t, err) t.Run("delete BFD", func(t *testing.T) { - err = nbClient.DeleteBFD(lrpName, dstIP1) + err = nbClient.DeleteBFDByDstIP(lrpName, dstIP1) require.NoError(t, err) bfdList, err := nbClient.ListBFDs(lrpName, dstIP1) @@ -144,7 +144,7 @@ func (suite *OvnClientTestSuite) testDeleteBFD() { }) t.Run("delete multiple BFDs", func(t *testing.T) { - err = nbClient.DeleteBFD(lrpName, "") + err = nbClient.DeleteBFDByDstIP(lrpName, "") require.NoError(t, err) bfdList, err := nbClient.ListBFDs(lrpName, "") @@ -155,16 +155,16 @@ func (suite *OvnClientTestSuite) testDeleteBFD() { t.Run("delete non-existent BFD", func(t *testing.T) { t.Parallel() - err := nbClient.DeleteBFD(lrpName, "192.168.124.17") + err := nbClient.DeleteBFDByDstIP(lrpName, "192.168.124.17") require.NoError(t, err) }) t.Run("closed server delete non-existent BFD", func(t *testing.T) { t.Parallel() - _, err := failedNbClient.CreateBFD(lrpName, dstIP1, minRx1, minTx1, detectMult1) + _, err := failedNbClient.CreateBFD(lrpName, dstIP1, minRx1, minTx1, detectMult1, nil) require.Error(t, err) - err = failedNbClient.DeleteBFD(lrpName, "192.168.124.17") + err = failedNbClient.DeleteBFDByDstIP(lrpName, "192.168.124.17") // cache db should be empty require.NoError(t, err) }) @@ -185,19 +185,19 @@ func (suite *OvnClientTestSuite) testListDownBFDs() { t.Run("list down BFDs", func(t *testing.T) { t.Parallel() - bfd1, err := nbClient.CreateBFD(lrpName, dstIP1, minRx, minTx, detectMult) + bfd1, err := nbClient.CreateBFD(lrpName, dstIP1, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd1) // closed server create failed BFD - failedBFD1, err := failedNbClient.CreateBFD(lrpName, dstIP1, minRx, minTx, detectMult) + failedBFD1, err := failedNbClient.CreateBFD(lrpName, dstIP1, minRx, minTx, detectMult, nil) require.Error(t, err) require.Nil(t, failedBFD1) - bfd2, err := nbClient.CreateBFD(lrpName, dstIP2, minRx, minTx, detectMult) + bfd2, err := nbClient.CreateBFD(lrpName, dstIP2, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd2) - bfd3, err := nbClient.CreateBFD(lrpName, dstIP3, minRx, minTx, detectMult) + bfd3, err := nbClient.CreateBFD(lrpName, dstIP3, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd3) @@ -247,7 +247,7 @@ func (suite *OvnClientTestSuite) testListDownBFDs() { t.Parallel() // Create a BFD with UP status - bfd, err := nbClient.CreateBFD(lrpName, "192.168.124.10", minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, "192.168.124.10", minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -277,15 +277,15 @@ func (suite *OvnClientTestSuite) testListUpBFDs() { t.Run("list up BFDs", func(t *testing.T) { t.Parallel() - bfd1, err := nbClient.CreateBFD(lrpName, dstIP1, minRx, minTx, detectMult) + bfd1, err := nbClient.CreateBFD(lrpName, dstIP1, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd1) - bfd2, err := nbClient.CreateBFD(lrpName, dstIP2, minRx, minTx, detectMult) + bfd2, err := nbClient.CreateBFD(lrpName, dstIP2, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd2) - bfd3, err := nbClient.CreateBFD(lrpName, dstIP3, minRx, minTx, detectMult) + bfd3, err := nbClient.CreateBFD(lrpName, dstIP3, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd3) @@ -340,7 +340,7 @@ func (suite *OvnClientTestSuite) testIsLrpBfdUp() { t.Parallel() lrpName := "test-is-lrp-bfd-up" - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -358,7 +358,7 @@ func (suite *OvnClientTestSuite) testIsLrpBfdUp() { t.Parallel() lrpName := "test-is-lrp-bfd-down" - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -377,7 +377,7 @@ func (suite *OvnClientTestSuite) testIsLrpBfdUp() { t.Parallel() lrpName := "test-is-lrp-bfd-status-nil" - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -414,7 +414,7 @@ func (suite *OvnClientTestSuite) testBfdAddL3HAHandler() { dstIP := "192.168.124.19" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -436,7 +436,7 @@ func (suite *OvnClientTestSuite) testBfdAddL3HAHandler() { dstIP := "192.168.124.20" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -460,7 +460,7 @@ func (suite *OvnClientTestSuite) testBfdAddL3HAHandler() { dstIP := "192.168.124.21" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -484,7 +484,7 @@ func (suite *OvnClientTestSuite) testBfdAddL3HAHandler() { dstIP := "192.168.124.22" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -510,7 +510,7 @@ func (suite *OvnClientTestSuite) testBfdUpdateL3HAHandler() { dstIP := "192.168.124.26" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -533,7 +533,7 @@ func (suite *OvnClientTestSuite) testBfdUpdateL3HAHandler() { dstIP := "192.168.124.27" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) bfd.Status = nil @@ -558,7 +558,7 @@ func (suite *OvnClientTestSuite) testBfdUpdateL3HAHandler() { dstIP := "192.168.124.28" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) upStatus := ovnnb.BFDStatusUp @@ -579,7 +579,7 @@ func (suite *OvnClientTestSuite) testBfdUpdateL3HAHandler() { dstIP := "192.168.124.23" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -602,7 +602,7 @@ func (suite *OvnClientTestSuite) testBfdUpdateL3HAHandler() { dstIP := "192.168.124.24" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -625,7 +625,7 @@ func (suite *OvnClientTestSuite) testBfdUpdateL3HAHandler() { dstIP := "192.168.124.25" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -648,7 +648,7 @@ func (suite *OvnClientTestSuite) testBfdUpdateL3HAHandler() { dstIP := "192.168.124.28" minRx, minTx, detectMult := 101, 102, 19 - failedBFD, err := failedNbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + failedBFD, err := failedNbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.Error(t, err) require.Nil(t, failedBFD) newBfd := &ovnnb.BFD{ @@ -677,7 +677,7 @@ func (suite *OvnClientTestSuite) testBfdDelL3HAHandler() { dstIP := "192.168.124.30" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) @@ -696,7 +696,7 @@ func (suite *OvnClientTestSuite) testBfdDelL3HAHandler() { dstIP := "192.168.124.31" minRx, minTx, detectMult := 101, 102, 19 - bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult) + bfd, err := nbClient.CreateBFD(lrpName, dstIP, minRx, minTx, detectMult, nil) require.NoError(t, err) require.NotNil(t, bfd) diff --git a/pkg/ovs/ovn-nb-ha_chassis_group.go b/pkg/ovs/ovn-nb-ha_chassis_group.go new file mode 100644 index 00000000000..44ac9eca98c --- /dev/null +++ b/pkg/ovs/ovn-nb-ha_chassis_group.go @@ -0,0 +1,177 @@ +package ovs + +import ( + "context" + "errors" + "fmt" + "maps" + "slices" + + "github.com/ovn-org/libovsdb/client" + "github.com/ovn-org/libovsdb/model" + "github.com/ovn-org/libovsdb/ovsdb" + "k8s.io/klog/v2" + + ovsclient "github.com/kubeovn/kube-ovn/pkg/ovsdb/client" + "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +// CreateHAChassisGroup adds or updates the ha chassis group +func (c *OVNNbClient) CreateHAChassisGroup(name string, chassises []string, externalIDs map[string]string) error { + group, err := c.GetHAChassisGroup(name, true) + if err != nil { + klog.Error(err) + return err + } + + var ops []ovsdb.Operation + if group == nil { + group = &ovnnb.HAChassisGroup{ + UUID: ovsclient.NamedUUID(), + Name: name, + ExternalIDs: map[string]string{"vendor": util.CniTypeName}, + } + maps.Insert(group.ExternalIDs, maps.All(externalIDs)) + createOps, err := c.Create(group) + if err != nil { + klog.Error(err) + return err + } + ops = append(ops, createOps...) + } else { + group.ExternalIDs = map[string]string{"vendor": util.CniTypeName} + maps.Insert(group.ExternalIDs, maps.All(externalIDs)) + updateOps, err := c.Where(group).Update(group, &group.ExternalIDs) + if err != nil { + klog.Error(err) + return err + } + ops = append(ops, updateOps...) + } + + ctx, cancel := context.WithTimeout(context.Background(), c.Timeout) + defer cancel() + + haChassises := make([]*ovnnb.HAChassis, 0, max(len(group.HaChassis), len(chassises))) + if len(group.HaChassis) != 0 { + if err = c.ovsDbClient.WhereCache(func(c *ovnnb.HAChassis) bool { + return slices.Contains(group.HaChassis, c.UUID) + }).List(ctx, &haChassises); err != nil { + klog.Error(err) + return err + } + } + + priorityMap := make(map[string]int, len(chassises)) + for i, chassis := range chassises { + priorityMap[chassis] = 100 - i + } + + uuids := make([]string, 0, len(group.HaChassis)) + for _, chassis := range haChassises { + if priority, ok := priorityMap[chassis.ChassisName]; ok { + delete(priorityMap, chassis.ChassisName) + if chassis.Priority != priority { + // update ha chassis priority + chassis.Priority = priority + updateOps, err := c.Where(chassis).Update(chassis, &chassis.Priority) + if err != nil { + klog.Error(err) + return err + } + ops = append(ops, updateOps...) + } + } else { + uuids = append(uuids, chassis.UUID) + } + } + if len(uuids) != 0 { + // delete ha chassis from the group + deleteOps, err := c.Where(group).Mutate(group, model.Mutation{ + Field: &group.HaChassis, + Value: uuids, + Mutator: ovsdb.MutateOperationDelete, + }) + if err != nil { + klog.Error(err) + return err + } + ops = append(ops, deleteOps...) + } + + // add new ha chassis to the group + for chassis, priority := range priorityMap { + haChassis := &ovnnb.HAChassis{ + UUID: ovsclient.NamedUUID(), + ChassisName: chassis, + Priority: priority, + ExternalIDs: map[string]string{"group": name, "vendor": util.CniTypeName}, + } + createOps, err := c.Create(haChassis) + if err != nil { + klog.Error(err) + return err + } + insertOps, err := c.Where(group).Mutate(group, model.Mutation{ + Field: &group.HaChassis, + Value: []string{haChassis.UUID}, + Mutator: ovsdb.MutateOperationInsert, + }) + if err != nil { + klog.Error(err) + return err + } + ops = append(ops, createOps...) + ops = append(ops, insertOps...) + } + + if err = c.Transact("ha-chassis-group-add", ops); err != nil { + err := fmt.Errorf("failed to add/update HA chassis group %s: %w", name, err) + klog.Error(err) + return err + } + return nil +} + +// GetHAChassisGroup gets the ha chassis group +func (c *OVNNbClient) GetHAChassisGroup(name string, ignoreNotFound bool) (*ovnnb.HAChassisGroup, error) { + ctx, cancel := context.WithTimeout(context.Background(), c.Timeout) + defer cancel() + + group := &ovnnb.HAChassisGroup{Name: name} + if err := c.Get(ctx, group); err != nil { + if ignoreNotFound && errors.Is(err, client.ErrNotFound) { + return nil, nil + } + + return nil, fmt.Errorf("failed to get HA chassis group %q: %w", name, err) + } + + return group, nil +} + +// DeleteHAChassisGroup deletes the ha chassis group +func (c *OVNNbClient) DeleteHAChassisGroup(name string) error { + group, err := c.GetHAChassisGroup(name, true) + if err != nil { + klog.Error(err) + return err + } + if group == nil { + return nil + } + + ops, err := c.Where(group).Delete() + if err != nil { + klog.Error(err) + return err + } + + if err = c.Transact("ha-chassis-group-del", ops); err != nil { + klog.Error(err) + return fmt.Errorf("failed to delete HA chassis group %q: %w", name, err) + } + + return nil +} diff --git a/pkg/ovs/ovn-nb-ha_chassis_group_test.go b/pkg/ovs/ovn-nb-ha_chassis_group_test.go new file mode 100644 index 00000000000..2f5fa07c201 --- /dev/null +++ b/pkg/ovs/ovn-nb-ha_chassis_group_test.go @@ -0,0 +1,96 @@ +package ovs + +import ( + "context" + "slices" + + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/util/uuid" + + "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +func (suite *OvnClientTestSuite) testCreateHAChassisGroup() { + t := suite.T() + t.Parallel() + + nbClient := suite.ovnNBClient + name := "test-create-ha-chassis-group" + chassises := []string{string(uuid.NewUUID()), string(uuid.NewUUID()), string(uuid.NewUUID())} + + err := nbClient.CreateHAChassisGroup(name, chassises, map[string]string{"k1": "v1"}) + require.NoError(t, err) + + group, err := nbClient.GetHAChassisGroup(name, false) + require.NoError(t, err) + require.NotNil(t, group) + require.Equal(t, group.ExternalIDs, map[string]string{"vendor": util.CniTypeName, "k1": "v1"}) + require.Len(t, group.HaChassis, len(chassises)) + for _, uuid := range group.HaChassis { + chassis := &ovnnb.HAChassis{UUID: uuid} + err = nbClient.Get(context.Background(), chassis) + require.NoError(t, err) + require.Contains(t, chassises, chassis.ChassisName) + require.Equal(t, chassis.Priority, 100-slices.Index(chassises, chassis.ChassisName)) + require.Equal(t, chassis.ExternalIDs, map[string]string{"group": name, "vendor": util.CniTypeName}) + } + + // update the ha chassis group + chassises[1], chassises[2] = chassises[2], string(uuid.NewUUID()) + err = nbClient.CreateHAChassisGroup(name, chassises, map[string]string{"k2": "v2"}) + require.NoError(t, err) + + group, err = nbClient.GetHAChassisGroup(name, false) + require.NoError(t, err) + require.NotNil(t, group) + require.Equal(t, group.ExternalIDs, map[string]string{"vendor": util.CniTypeName, "k2": "v2"}) + require.Len(t, group.HaChassis, len(chassises)) + for _, uuid := range group.HaChassis { + chassis := &ovnnb.HAChassis{UUID: uuid} + err = nbClient.Get(context.Background(), chassis) + require.NoError(t, err) + require.Contains(t, chassises, chassis.ChassisName) + require.Equal(t, chassis.Priority, 100-slices.Index(chassises, chassis.ChassisName)) + require.Equal(t, chassis.ExternalIDs, map[string]string{"group": name, "vendor": util.CniTypeName}) + } +} + +func (suite *OvnClientTestSuite) testGetHAChassisGroup() { + t := suite.T() + t.Parallel() + + nbClient := suite.ovnNBClient + name := "non-existent-ha-chassis-group" + + group, err := nbClient.GetHAChassisGroup(name, true) + require.NoError(t, err) + require.Nil(t, group) + + group, err = nbClient.GetHAChassisGroup(name, false) + require.Error(t, err) + require.Nil(t, group) +} + +func (suite *OvnClientTestSuite) testDeleteHAChassisGroup() { + t := suite.T() + t.Parallel() + + nbClient := suite.ovnNBClient + name := "test-delete-ha-chassis-group" + chassises := []string{string(uuid.NewUUID()), string(uuid.NewUUID()), string(uuid.NewUUID())} + + err := nbClient.CreateHAChassisGroup(name, chassises, nil) + require.NoError(t, err) + + group, err := nbClient.GetHAChassisGroup(name, false) + require.NoError(t, err) + require.NotNil(t, group) + + err = nbClient.DeleteHAChassisGroup(name) + require.NoError(t, err) + + group, err = nbClient.GetHAChassisGroup(name, true) + require.NoError(t, err) + require.Nil(t, group) +} diff --git a/pkg/ovs/ovn-nb-logical_router_policy.go b/pkg/ovs/ovn-nb-logical_router_policy.go index c3eaee2cef5..97f9a10ba84 100644 --- a/pkg/ovs/ovn-nb-logical_router_policy.go +++ b/pkg/ovs/ovn-nb-logical_router_policy.go @@ -19,7 +19,7 @@ import ( ) // AddLogicalRouterPolicy add a policy route to logical router -func (c *OVNNbClient) AddLogicalRouterPolicy(lrName string, priority int, match, action string, nextHops []string, externalIDs map[string]string) error { +func (c *OVNNbClient) AddLogicalRouterPolicy(lrName string, priority int, match, action string, nextHops, bfdSessions []string, externalIDs map[string]string) error { fnFilter := func(policy *ovnnb.LogicalRouterPolicy) bool { return policy.Priority == priority && policy.Match == match } @@ -52,7 +52,7 @@ func (c *OVNNbClient) AddLogicalRouterPolicy(lrName string, priority int, match, if policyFound == nil { klog.Infof("creating lr policy with priority = %d, match = %q, action = %q, nextHops = %q", priority, match, action, nextHops) - policy := c.newLogicalRouterPolicy(priority, match, action, nextHops, externalIDs) + policy := c.newLogicalRouterPolicy(priority, match, action, nextHops, bfdSessions, externalIDs) if err := c.CreateLogicalRouterPolicies(lrName, policy); err != nil { klog.Error(err) return fmt.Errorf("add policy to logical router %s: %w", lrName, err) @@ -281,13 +281,14 @@ func (c *OVNNbClient) ListLogicalRouterPolicies(lrName string, priority int, ext } // newLogicalRouterPolicy return logical router policy with basic information -func (c *OVNNbClient) newLogicalRouterPolicy(priority int, match, action string, nextHops []string, externalIDs map[string]string) *ovnnb.LogicalRouterPolicy { +func (c *OVNNbClient) newLogicalRouterPolicy(priority int, match, action string, nextHops, bfdSessions []string, externalIDs map[string]string) *ovnnb.LogicalRouterPolicy { return &ovnnb.LogicalRouterPolicy{ UUID: ovsclient.NamedUUID(), Priority: priority, Match: match, Action: action, Nexthops: nextHops, + BFDSessions: bfdSessions, ExternalIDs: externalIDs, } } @@ -324,6 +325,19 @@ func policyFilter(priority int, externalIDs map[string]string, ignoreExtIDEmptyV } } +func (c *OVNNbClient) UpdateLogicalRouterPolicy(policy *ovnnb.LogicalRouterPolicy, fields ...interface{}) error { + ops, err := c.ovsDbClient.Where(policy).Update(policy, fields...) + if err != nil { + klog.Error(err) + return fmt.Errorf("failed to generate update operations for logical router policy %s: %w", policy.UUID, err) + } + if err = c.Transact("lr-policy-update", ops); err != nil { + klog.Error(err) + return fmt.Errorf("failed to update logical router policy %s: %w", policy.UUID, err) + } + return nil +} + func (c *OVNNbClient) DeleteRouterPolicy(lr *ovnnb.LogicalRouter, uuid string) error { ops, err := c.ovsDbClient.Where(lr).Mutate(lr, model.Mutation{ Field: &lr.Policies, @@ -336,7 +350,7 @@ func (c *OVNNbClient) DeleteRouterPolicy(lr *ovnnb.LogicalRouter, uuid string) e } if err = c.Transact("lr-policy-delete", ops); err != nil { klog.Error(err) - return fmt.Errorf("failed to delete route policy %s: %w", uuid, err) + return fmt.Errorf("failed to delete router policy %s: %w", uuid, err) } return nil } diff --git a/pkg/ovs/ovn-nb-logical_router_policy_test.go b/pkg/ovs/ovn-nb-logical_router_policy_test.go index 61f87e2c702..e404a1a4858 100644 --- a/pkg/ovs/ovn-nb-logical_router_policy_test.go +++ b/pkg/ovs/ovn-nb-logical_router_policy_test.go @@ -36,7 +36,7 @@ func (suite *OvnClientTestSuite) testAddLogicalRouterPolicy() { err := nbClient.CreateLogicalRouter(lrName) require.NoError(t, err) - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, nil) require.NoError(t, err) lr, err := nbClient.GetLogicalRouter(lrName, false) @@ -48,28 +48,28 @@ func (suite *OvnClientTestSuite) testAddLogicalRouterPolicy() { require.Contains(t, lr.Policies, policyList[0].UUID) t.Run("normal add policy", func(t *testing.T) { - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, nil) require.NoError(t, err) action = ovnnb.LogicalRouterPolicyActionDrop - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, nil) require.NoError(t, err) }) t.Run("should log err when logical router does not exist", func(t *testing.T) { - err = nbClient.AddLogicalRouterPolicy("test-nonexist-lr", priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy("test-nonexist-lr", priority, match, action, nextHops, nil, nil) require.Error(t, err) }) t.Run("handle duplicate policies with matching action and nextHops", func(t *testing.T) { - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, nil) require.NoError(t, err) - duplicatePolicy := nbClient.newLogicalRouterPolicy(priority, match, action, nextHops, nil) + duplicatePolicy := nbClient.newLogicalRouterPolicy(priority, match, action, nextHops, nil, nil) err = nbClient.CreateLogicalRouterPolicies(lrName, duplicatePolicy) require.NoError(t, err) - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, nil) require.NoError(t, err) finalPolicyList, err := nbClient.GetLogicalRouterPolicy(lrName, priority, match, false) @@ -80,7 +80,7 @@ func (suite *OvnClientTestSuite) testAddLogicalRouterPolicy() { t.Run("update policy with different externalIDs", func(t *testing.T) { initialExternalIDs := map[string]string{"key1": "value1"} - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, initialExternalIDs) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, initialExternalIDs) require.NoError(t, err) policyList, err := nbClient.GetLogicalRouterPolicy(lrName, priority, match, false) @@ -90,7 +90,7 @@ func (suite *OvnClientTestSuite) testAddLogicalRouterPolicy() { newExternalIDs := map[string]string{"key2": "value2"} - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, newExternalIDs) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, newExternalIDs) require.NoError(t, err) updatedPolicyList, err := nbClient.GetLogicalRouterPolicy(lrName, priority, match, false) @@ -119,7 +119,7 @@ func (suite *OvnClientTestSuite) testCreateLogicalRouterPolicies() { t.Run("add policies to logical router", func(t *testing.T) { for i := 0; i < 3; i++ { match := fmt.Sprintf("%s && tcp.dst == %d", matchPrefix, basePort+i) - policy := nbClient.newLogicalRouterPolicy(priority, match, action, nil, nil) + policy := nbClient.newLogicalRouterPolicy(priority, match, action, nil, nil, nil) policies = append(policies, policy) } @@ -170,7 +170,7 @@ func (suite *OvnClientTestSuite) testDeleteLogicalRouterPolicy() { err := nbClient.CreateLogicalRouter(lrName) require.NoError(t, err) - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, nil) require.NoError(t, err) t.Run("no err when delete existent logical switch port", func(t *testing.T) { @@ -222,7 +222,7 @@ func (suite *OvnClientTestSuite) testDeleteLogicalRouterPolicies() { prepare := func() { for i := 0; i < 3; i++ { priority := basePriority + i - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, externalIDs) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, externalIDs) require.NoError(t, err) } @@ -296,7 +296,7 @@ func (suite *OvnClientTestSuite) testClearLogicalRouterPolicy() { for i := 0; i < 3; i++ { priority := basePriority + i - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, nil) require.NoError(t, err) } @@ -342,7 +342,7 @@ func (suite *OvnClientTestSuite) testGetLogicalRouterPolicy() { err := nbClient.CreateLogicalRouter(lrName) require.NoError(t, err) - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, ovnnb.LogicalRouterPolicyActionAllow, nil, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, ovnnb.LogicalRouterPolicyActionAllow, nil, nil, nil) require.NoError(t, err) t.Run("priority and match are same", func(t *testing.T) { @@ -409,7 +409,7 @@ func (suite *OvnClientTestSuite) testGetLogicalRouterPolicyByUUID() { err := nbClient.CreateLogicalRouter(lrName) require.NoError(t, err) - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, nil) require.NoError(t, err) lr, err := nbClient.GetLogicalRouter(lrName, false) @@ -453,7 +453,7 @@ func (suite *OvnClientTestSuite) testGetLogicalRouterPolicyByExtID() { err := nbClient.CreateLogicalRouter(lrName) require.NoError(t, err) - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, extID) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, extID) require.NoError(t, err) lr, err := nbClient.GetLogicalRouter(lrName, false) @@ -491,7 +491,7 @@ func (suite *OvnClientTestSuite) testGetLogicalRouterPolicyByExtID() { t.Run("get lrp with empty ExternalIDs", func(t *testing.T) { t.Parallel() - emptyExtIDPolicy := nbClient.newLogicalRouterPolicy(priority, match, action, nextHops, nil) + emptyExtIDPolicy := nbClient.newLogicalRouterPolicy(priority, match, action, nextHops, nil, nil) err = nbClient.CreateLogicalRouterPolicies(lrName, emptyExtIDPolicy) require.NoError(t, err) @@ -523,7 +523,7 @@ func (suite *OvnClientTestSuite) testNewLogicalRouterPolicy() { ExternalIDs: map[string]string{"key": "value"}, } - policy := nbClient.newLogicalRouterPolicy(priority, match, action, nextHops, map[string]string{"key": "value"}) + policy := nbClient.newLogicalRouterPolicy(priority, match, action, nextHops, nil, map[string]string{"key": "value"}) expect.UUID = policy.UUID require.Equal(t, expect, policy) } @@ -622,7 +622,7 @@ func (suite *OvnClientTestSuite) testDeleteRouterPolicy() { err := nbClient.CreateLogicalRouter(lrName) require.NoError(t, err) - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, nil) require.NoError(t, err) lr, err := nbClient.GetLogicalRouter(lrName, false) @@ -651,7 +651,7 @@ func (suite *OvnClientTestSuite) testDeleteLogicalRouterPolicyByNexthop() { err := nbClient.CreateLogicalRouter(lrName) require.NoError(t, err) - err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil) + err = nbClient.AddLogicalRouterPolicy(lrName, priority, match, action, nextHops, nil, nil) require.NoError(t, err) lr, err := nbClient.GetLogicalRouter(lrName, false) diff --git a/pkg/ovs/ovn-nb-logical_router_port.go b/pkg/ovs/ovn-nb-logical_router_port.go index 47778b3de61..bd2e2a217a9 100644 --- a/pkg/ovs/ovn-nb-logical_router_port.go +++ b/pkg/ovs/ovn-nb-logical_router_port.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "maps" + "slices" "strings" "github.com/ovn-org/libovsdb/client" @@ -83,6 +85,20 @@ func (c *OVNNbClient) UpdateLogicalRouterPortRA(lrpName, ipv6RAConfigsStr string return c.UpdateLogicalRouterPort(lrp, &lrp.Ipv6Prefix, &lrp.Ipv6RaConfigs) } +func (c *OVNNbClient) UpdateLogicalRouterPortNetworks(lrpName string, networks []string) error { + lrp, err := c.GetLogicalRouterPort(lrpName, false) + if err != nil { + klog.Error(err) + return err + } + if slices.Equal(networks, lrp.Networks) { + return nil + } + + lrp.Networks = slices.Clone(networks) + return c.UpdateLogicalRouterPort(lrp, &lrp.Networks) +} + func (c *OVNNbClient) UpdateLogicalRouterPortOptions(lrpName string, options map[string]string) error { if len(options) == 0 { return nil @@ -94,20 +110,41 @@ func (c *OVNNbClient) UpdateLogicalRouterPortOptions(lrpName string, options map return err } + newOptions := maps.Clone(lrp.Options) for k, v := range options { if len(v) == 0 { - delete(lrp.Options, k) + delete(newOptions, k) } else { - if len(lrp.Options) == 0 { - lrp.Options = make(map[string]string) + if len(newOptions) == 0 { + newOptions = make(map[string]string) } - lrp.Options[k] = v + newOptions[k] = v } } + if maps.Equal(newOptions, lrp.Options) { + return nil + } + lrp.Options = newOptions return c.UpdateLogicalRouterPort(lrp, &lrp.Options) } +func (c *OVNNbClient) SetLogicalRouterPortHAChassisGroup(lrpName, haChassisGroupName string) error { + lrp, err := c.GetLogicalRouterPort(lrpName, false) + if err != nil { + klog.Error(err) + return err + } + group, err := c.GetHAChassisGroup(haChassisGroupName, false) + if err != nil { + klog.Error(err) + return err + } + + lrp.HaChassisGroup = &group.UUID + return c.UpdateLogicalRouterPort(lrp, &lrp.HaChassisGroup) +} + // UpdateLogicalRouterPort update logical router port func (c *OVNNbClient) UpdateLogicalRouterPort(lrp *ovnnb.LogicalRouterPort, fields ...interface{}) error { if lrp == nil { diff --git a/pkg/ovs/ovn-nb-logical_router_route.go b/pkg/ovs/ovn-nb-logical_router_route.go index fd5a393ab10..6138c1ce89a 100644 --- a/pkg/ovs/ovn-nb-logical_router_route.go +++ b/pkg/ovs/ovn-nb-logical_router_route.go @@ -184,6 +184,67 @@ func (c *OVNNbClient) DeleteLogicalRouterStaticRoute(lrName string, routeTable, return nil } +// DeleteLogicalRouterStaticRoute delete a logical router static route +func (c *OVNNbClient) DeleteLogicalRouterStaticRouteByUUID(lrName, uuid string) error { + lr, err := c.GetLogicalRouter(lrName, true) + if err != nil { + return err + } + if lr == nil { + return nil + } + + // remove static route from logical router + ops, err := c.LogicalRouterUpdateStaticRouteOp(lrName, []string{uuid}, ovsdb.MutateOperationDelete) + if err != nil { + klog.Error(err) + return fmt.Errorf("generate operations for removing static route %s from logical router %s: %w", uuid, lrName, err) + } + if err = c.Transact("lr-route-del", ops); err != nil { + klog.Error(err) + return fmt.Errorf("delete static route %s from logical router %s: %w", uuid, lrName, err) + } + + return nil +} + +func (c *OVNNbClient) DeleteLogicalRouterStaticRouteByExternalIDs(lrName string, externalIDs map[string]string) error { + lr, err := c.GetLogicalRouter(lrName, true) + if err != nil { + return err + } + if lr == nil { + return nil + } + + routes, err := c.ListLogicalRouterStaticRoutes(lrName, nil, nil, "", externalIDs) + if err != nil { + klog.Error(err) + return err + } + if len(routes) == 0 { + return nil + } + + uuids := make([]string, 0, len(routes)) + for _, route := range routes { + uuids = append(uuids, route.UUID) + } + + // remove static route from logical router + ops, err := c.LogicalRouterUpdateStaticRouteOp(lrName, uuids, ovsdb.MutateOperationDelete) + if err != nil { + klog.Error(err) + return fmt.Errorf("generate operations for removing static routes %v from logical router %s: %w", uuids, lrName, err) + } + if err = c.Transact("lr-route-del", ops); err != nil { + klog.Error(err) + return fmt.Errorf("delete static routes %v from logical router %s: %w", uuids, lrName, err) + } + + return nil +} + // ClearLogicalRouterStaticRoute clear static route from logical router once func (c *OVNNbClient) ClearLogicalRouterStaticRoute(lrName string) error { lr, err := c.GetLogicalRouter(lrName, false) diff --git a/pkg/ovs/ovn-nb-suite_test.go b/pkg/ovs/ovn-nb-suite_test.go index 088725304ad..054d39f8607 100644 --- a/pkg/ovs/ovn-nb-suite_test.go +++ b/pkg/ovs/ovn-nb-suite_test.go @@ -496,6 +496,19 @@ func (suite *OvnClientTestSuite) Test_NewGatewayChassis() { suite.testNewGatewayChassis() } +/* ha_chassis_group unit test */ +func (suite *OvnClientTestSuite) Test_CreateHAChassisGroup() { + suite.testCreateHAChassisGroup() +} + +func (suite *OvnClientTestSuite) Test_GetHAChassisGroup() { + suite.testGetHAChassisGroup() +} + +func (suite *OvnClientTestSuite) Test_DeleteHAChassisGroup() { + suite.testDeleteHAChassisGroup() +} + /* load_balancer unit test */ func (suite *OvnClientTestSuite) Test_CreateLoadBalancer() { suite.testCreateLoadBalancer() @@ -1337,6 +1350,8 @@ func newNbClient(addr string, timeout int) (client.Client, error) { client.WithTable(&ovnnb.BFD{}), client.WithTable(&ovnnb.DHCPOptions{}), client.WithTable(&ovnnb.GatewayChassis{}), + client.WithTable(&ovnnb.HAChassis{}), + client.WithTable(&ovnnb.HAChassisGroup{}), client.WithTable(&ovnnb.LoadBalancer{}), client.WithTable(&ovnnb.LoadBalancerHealthCheck{}), client.WithTable(&ovnnb.LogicalRouterPolicy{}), diff --git a/pkg/ovs/ovn.go b/pkg/ovs/ovn.go index c1baeba2a03..2a0f6f22e8c 100644 --- a/pkg/ovs/ovn.go +++ b/pkg/ovs/ovn.go @@ -44,6 +44,9 @@ const ( IfExists = "--if-exists" OVSDBWaitTimeout = 0 + + ExternalIDVendor = "vendor" + ExternalIDVpcEgressGateway = "vpc-egress-gateway" ) // NewLegacyClient init a legacy ovn client @@ -66,6 +69,8 @@ func NewOvnNbClient(ovnNbAddr string, ovnNbTimeout, ovsDbConTimeout, ovsDbInacti client.WithTable(&ovnnb.BFD{}), client.WithTable(&ovnnb.DHCPOptions{}), client.WithTable(&ovnnb.GatewayChassis{}), + client.WithTable(&ovnnb.HAChassis{}), + client.WithTable(&ovnnb.HAChassisGroup{}), client.WithTable(&ovnnb.LoadBalancer{}), client.WithTable(&ovnnb.LoadBalancerHealthCheck{}), client.WithTable(&ovnnb.LogicalRouterPolicy{}), diff --git a/pkg/util/const.go b/pkg/util/const.go index 8acda2daaf0..1b9974e8488 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -48,6 +48,9 @@ const ( LogicalRouterAnnotation = "ovn.kubernetes.io/logical_router" VpcAnnotation = "ovn.kubernetes.io/vpc" + VpcEgressGatewayAnnotation = "ovn.kubernetes.io/vpc_egress_gateway" + GenerateHashAnnotation = "ovn.kubernetes.io/generate-hash" + Layer2ForwardAnnotationTemplate = "%s.kubernetes.io/layer2_forward" PortSecurityAnnotationTemplate = "%s.kubernetes.io/port_security" PortVipAnnotationTemplate = "%s.kubernetes.io/port_vips" @@ -210,6 +213,7 @@ const ( U2OSubnetPolicyPriority = 29400 GatewayRouterPolicyPriority = 29000 + EgressGatewayPolicyPriority = 29100 NorthGatewayRoutePolicyPriority = 29250 OvnICPolicyPriority = 29500 NodeRouterPolicyPriority = 30000 diff --git a/pkg/util/hash.go b/pkg/util/hash.go new file mode 100644 index 00000000000..da753e9740d --- /dev/null +++ b/pkg/util/hash.go @@ -0,0 +1,26 @@ +package util + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" +) + +func Sha256Hash(input []byte) string { + hasher := sha256.New() + hasher.Write(input) + hashedBytes := hasher.Sum(nil) + return hex.EncodeToString(hashedBytes) +} + +func Sha256HashObject(obj any) (string, error) { + buf, err := json.Marshal(obj) + if err != nil { + return "", err + } + + hasher := sha256.New() + hasher.Write(buf) + hashedBytes := hasher.Sum(nil) + return hex.EncodeToString(hashedBytes), nil +} diff --git a/pkg/util/k8s.go b/pkg/util/k8s.go index 61eddeac990..c12ff35a558 100644 --- a/pkg/util/k8s.go +++ b/pkg/util/k8s.go @@ -10,6 +10,7 @@ import ( "strings" "time" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -17,6 +18,9 @@ import ( "k8s.io/apimachinery/pkg/types" clientv1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/scheme" ) func DialTCP(host string, timeout time.Duration, verbose bool) error { @@ -152,3 +156,29 @@ func nodeMergePatch(cs clientv1.NodeInterface, node, patch string) error { } return nil } + +func SetOwnerReference(owner, object metav1.Object) error { + return controllerutil.SetOwnerReference(owner, object, scheme.Scheme) +} + +func DeploymentIsReady(deployment *appsv1.Deployment) bool { + if deployment.Generation > deployment.Status.ObservedGeneration { + return false + } + + for _, condition := range deployment.Status.Conditions { + if condition.Type == appsv1.DeploymentProgressing { + // deployment exceeded its progress deadline + if condition.Reason == "ProgressDeadlineExceeded" { + return false + } + break + } + } + if (deployment.Spec.Replicas != nil && deployment.Status.UpdatedReplicas < *deployment.Spec.Replicas) || + deployment.Status.Replicas > deployment.Status.UpdatedReplicas || + deployment.Status.AvailableReplicas < deployment.Status.UpdatedReplicas { + return false + } + return true +} diff --git a/pkg/util/strings.go b/pkg/util/strings.go index 81b10c50ee6..ac3d087f17b 100644 --- a/pkg/util/strings.go +++ b/pkg/util/strings.go @@ -1,8 +1,6 @@ package util import ( - "crypto/sha256" - "encoding/hex" "strings" ) @@ -27,10 +25,3 @@ func DoubleQuotedFields(s string) []string { return fields } - -func Sha256Hash(input []byte) string { - hasher := sha256.New() - hasher.Write(input) - hashedBytes := hasher.Sum(nil) - return hex.EncodeToString(hashedBytes) -} diff --git a/test/anp/anp_test.go b/test/anp/anp_test.go index d159d17197e..7dd727a8fe7 100644 --- a/test/anp/anp_test.go +++ b/test/anp/anp_test.go @@ -82,7 +82,7 @@ func TestAdminNetworkPolicyConformance(t *testing.T) { if err != nil { t.Fatalf("error marshalling conformance profile report: %v", err) } - err = os.WriteFile("../../"+reportFileName, rawReport, 0600) + err = os.WriteFile("../../"+reportFileName, rawReport, 0o600) if err != nil { t.Fatalf("error writing conformance profile report: %v", err) } diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 11777e1d787..b2923a0853a 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -34,6 +34,17 @@ const ( timeout = 2 * time.Minute ) +func LoadKubeOVNClientSet() (*kubeovncs.Clientset, error) { + config, err := framework.LoadConfig() + if err != nil { + return nil, err + } + + config.QPS = 20 + config.Burst = 50 + return kubeovncs.NewForConfig(config) +} + type Framework struct { KubeContext string *framework.Framework diff --git a/test/e2e/framework/provider-network.go b/test/e2e/framework/provider-network.go index 76fdc007e7b..1c500ea5f83 100644 --- a/test/e2e/framework/provider-network.go +++ b/test/e2e/framework/provider-network.go @@ -18,6 +18,7 @@ import ( "github.com/onsi/gomega" apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + clientset "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" v1 "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/typed/kubeovn/v1" "github.com/kubeovn/kube-ovn/pkg/util" ) @@ -28,6 +29,12 @@ type ProviderNetworkClient struct { v1.ProviderNetworkInterface } +func NewProviderNetworkClient(cs clientset.Interface) *ProviderNetworkClient { + return &ProviderNetworkClient{ + ProviderNetworkInterface: cs.KubeovnV1().ProviderNetworks(), + } +} + func (f *Framework) ProviderNetworkClient() *ProviderNetworkClient { return &ProviderNetworkClient{ f: f, diff --git a/test/e2e/framework/vlan.go b/test/e2e/framework/vlan.go index 0da08795669..75e3bf8b19d 100644 --- a/test/e2e/framework/vlan.go +++ b/test/e2e/framework/vlan.go @@ -13,6 +13,7 @@ import ( "github.com/onsi/ginkgo/v2" apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + clientset "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" v1 "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/typed/kubeovn/v1" "github.com/kubeovn/kube-ovn/pkg/util" ) @@ -23,6 +24,12 @@ type VlanClient struct { v1.VlanInterface } +func NewVlanClient(cs clientset.Interface) *VlanClient { + return &VlanClient{ + VlanInterface: cs.KubeovnV1().Vlans(), + } +} + func (f *Framework) VlanClient() *VlanClient { return &VlanClient{ f: f, @@ -74,9 +81,9 @@ func (c *VlanClient) Patch(original, modified *apiv1.Vlan, timeout time.Duration } // Delete deletes a vlan if the vlan exists -func (c *VlanClient) Delete(name string, options metav1.DeleteOptions) { +func (c *VlanClient) Delete(name string) { ginkgo.GinkgoHelper() - err := c.VlanInterface.Delete(context.TODO(), name, options) + err := c.VlanInterface.Delete(context.TODO(), name, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { Failf("Failed to delete vlan %q: %v", name, err) } diff --git a/test/e2e/framework/vpc-egress-gateway.go b/test/e2e/framework/vpc-egress-gateway.go new file mode 100644 index 00000000000..3d82730c988 --- /dev/null +++ b/test/e2e/framework/vpc-egress-gateway.go @@ -0,0 +1,189 @@ +package framework + +import ( + "context" + "errors" + "fmt" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/test/e2e/framework" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + clientset "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" + v1 "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/typed/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +// VpcEgressGatewayClient is a struct for vpc egress gateway client. +type VpcEgressGatewayClient struct { + f *Framework + namespace string + v1.VpcEgressGatewayInterface +} + +func NewVpcEgressGatewayClient(cs clientset.Interface, namespapce string) *VpcEgressGatewayClient { + return &VpcEgressGatewayClient{ + namespace: namespapce, + VpcEgressGatewayInterface: cs.KubeovnV1().VpcEgressGateways(namespapce), + } +} + +func (f *Framework) VpcEgressGatewayClient() *VpcEgressGatewayClient { + return &VpcEgressGatewayClient{ + f: f, + namespace: f.Namespace.Name, + VpcEgressGatewayInterface: f.KubeOVNClientSet.KubeovnV1().VpcEgressGateways(f.Namespace.Name), + } +} + +func (f *Framework) VpcEgressGatewayClientNS(namespapce string) *VpcEgressGatewayClient { + return &VpcEgressGatewayClient{ + f: f, + namespace: namespapce, + VpcEgressGatewayInterface: f.KubeOVNClientSet.KubeovnV1().VpcEgressGateways(namespapce), + } +} + +func (c *VpcEgressGatewayClient) Get(name string) *apiv1.VpcEgressGateway { + ginkgo.GinkgoHelper() + gateway, err := c.VpcEgressGatewayInterface.Get(context.TODO(), name, metav1.GetOptions{}) + ExpectNoError(err) + return gateway +} + +// Create creates a new vpc-egress-gateway according to the framework specifications +func (c *VpcEgressGatewayClient) Create(gateway *apiv1.VpcEgressGateway) *apiv1.VpcEgressGateway { + ginkgo.GinkgoHelper() + g, err := c.VpcEgressGatewayInterface.Create(context.TODO(), gateway, metav1.CreateOptions{}) + ExpectNoError(err, "Error creating vpc-egress-gateway") + return g.DeepCopy() +} + +// CreateSync creates a new vpc-egress-gateway according to the framework specifications, and waits for it to be ready. +func (c *VpcEgressGatewayClient) CreateSync(gateway *apiv1.VpcEgressGateway) *apiv1.VpcEgressGateway { + ginkgo.GinkgoHelper() + _ = c.Create(gateway) + return c.WaitUntil(gateway.Name, func(g *apiv1.VpcEgressGateway) (bool, error) { + return g.Ready(), nil + }, "", 2*time.Second, timeout) +} + +// Patch patches the gateway +func (c *VpcEgressGatewayClient) Patch(original, modified *apiv1.VpcEgressGateway) *apiv1.VpcEgressGateway { + ginkgo.GinkgoHelper() + + patch, err := util.GenerateMergePatchPayload(original, modified) + ExpectNoError(err) + + var patchedGateway *apiv1.VpcEgressGateway + err = wait.PollUntilContextTimeout(context.Background(), 2*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + g, err := c.VpcEgressGatewayInterface.Patch(ctx, original.Name, types.MergePatchType, patch, metav1.PatchOptions{}, "") + if err != nil { + return handleWaitingAPIError(err, false, "patch vpc-egress-gateway %s/%s", original.Namespace, original.Name) + } + patchedGateway = g + return true, nil + }) + if err == nil { + return patchedGateway.DeepCopy() + } + + if errors.Is(err, context.DeadlineExceeded) { + Failf("timed out while retrying to patch vpc-egress-gateway %s/%s", original.Namespace, original.Name) + } + Failf("error occurred while retrying to patch vpc-egress-gateway %s/%s: %v", original.Namespace, original.Name, err) + + return nil +} + +// PatchSync patches the gateway and waits the gateway to meet the condition +func (c *VpcEgressGatewayClient) PatchSync(original, modified *apiv1.VpcEgressGateway) *apiv1.VpcEgressGateway { + ginkgo.GinkgoHelper() + _ = c.Patch(original, modified) + return c.WaitUntil(original.Name, func(g *apiv1.VpcEgressGateway) (bool, error) { + return g.Ready(), nil + }, "", 2*time.Second, timeout) +} + +// Delete deletes a vpc-egress-gateway if the vpc-egress-gateway exists +func (c *VpcEgressGatewayClient) Delete(name string) { + ginkgo.GinkgoHelper() + err := c.VpcEgressGatewayInterface.Delete(context.TODO(), name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + Failf("Failed to delete vpc-egress-gateway %s/%s: %v", c.namespace, name, err) + } +} + +// DeleteSync deletes the vpc-egress-gateway and waits for the vpc-egress-gateway to disappear for `timeout`. +// If the vpc-egress-gateway doesn't disappear before the timeout, it will fail the test. +func (c *VpcEgressGatewayClient) DeleteSync(name string) { + ginkgo.GinkgoHelper() + c.Delete(name) + gomega.Expect(c.WaitToDisappear(name, 2*time.Second, timeout)).To(gomega.Succeed(), "wait for vpc-egress-gateway %s/%s to disappear", c.namespace, name) +} + +// WaitUntil waits the given timeout duration for the specified condition to be met. +func (c *VpcEgressGatewayClient) WaitUntil(name string, cond func(g *apiv1.VpcEgressGateway) (bool, error), condDesc string, interval, timeout time.Duration) *apiv1.VpcEgressGateway { + var gateway *apiv1.VpcEgressGateway + err := wait.PollUntilContextTimeout(context.Background(), interval, timeout, false, func(_ context.Context) (bool, error) { + Logf("Waiting for vpc-egress-gateway %s/%s to meet condition %q", c.namespace, name, condDesc) + gateway = c.Get(name).DeepCopy() + met, err := cond(gateway) + if err != nil { + return false, fmt.Errorf("failed to check condition for vpc-egress-gateway %s/%s: %w", c.namespace, name, err) + } + if met { + Logf("vpc-egress-gateway %s/%s met condition %q", c.namespace, name, condDesc) + } else { + Logf("vpc-egress-gateway %s/%s not met condition %q", c.namespace, name, condDesc) + } + return met, nil + }) + if err == nil { + return gateway + } + + if errors.Is(err, context.DeadlineExceeded) { + Failf("timed out while waiting for vpc-egress-gateway %s/%s to meet condition %q", c.namespace, name, condDesc) + } + Failf("error occurred while waiting for vpc-egress-gateway %s/%s to meet condition %q: %v", c.namespace, name, condDesc, err) + + return nil +} + +// WaitToDisappear waits the given timeout duration for the specified vpc-egress-gateway to disappear. +func (c *VpcEgressGatewayClient) WaitToDisappear(name string, _, timeout time.Duration) error { + err := framework.Gomega().Eventually(context.Background(), framework.HandleRetry(func(ctx context.Context) (*apiv1.VpcEgressGateway, error) { + svc, err := c.VpcEgressGatewayInterface.Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil, nil + } + return svc, err + })).WithTimeout(timeout).Should(gomega.BeNil()) + if err != nil { + return fmt.Errorf("expected vpc-egress-gateway %s/%s to not be found: %w", c.namespace, name, err) + } + return nil +} + +func MakeVpcEgressGateway(namespace, name, vpc string, replicas int32, internalSubnet, externalSubnet string) *apiv1.VpcEgressGateway { + return &apiv1.VpcEgressGateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: apiv1.VpcEgressGatewaySpec{ + Replicas: replicas, + VPC: vpc, + InternalSubnet: internalSubnet, + ExternalSubnet: externalSubnet, + }, + } +} diff --git a/test/e2e/framework/vpc.go b/test/e2e/framework/vpc.go index f6c897a2438..8e72b0ef30b 100644 --- a/test/e2e/framework/vpc.go +++ b/test/e2e/framework/vpc.go @@ -89,7 +89,7 @@ func (c *VpcClient) Patch(original, modified *kubeovnv1.Vpc) *kubeovnv1.Vpc { // PatchSync patches the vpc and waits for the vpc to be ready for `timeout`. // If the vpc doesn't become ready before the timeout, it will fail the test. -func (c *VpcClient) PatchSync(original, modified *kubeovnv1.Vpc, _ []string, timeout time.Duration) *kubeovnv1.Vpc { +func (c *VpcClient) PatchSync(original, modified *kubeovnv1.Vpc, timeout time.Duration) *kubeovnv1.Vpc { ginkgo.GinkgoHelper() vpc := c.Patch(original, modified) diff --git a/test/e2e/kube-ovn/underlay/underlay.go b/test/e2e/kube-ovn/underlay/underlay.go index d616e61040c..b4e43c46138 100644 --- a/test/e2e/kube-ovn/underlay/underlay.go +++ b/test/e2e/kube-ovn/underlay/underlay.go @@ -12,7 +12,6 @@ import ( dockernetwork "github.com/docker/docker/api/types/network" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epodoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" @@ -208,7 +207,7 @@ var _ = framework.SerialDescribe("[group:underlay]", func() { routeMap[node.ID] = append(routeMap[node.ID], r) } } - framework.ExpectHaveKey(linkMap, node.ID) + framework.ExpectHaveKey(routeMap, node.ID) linkMap[node.Name()] = linkMap[node.ID] routeMap[node.Name()] = routeMap[node.ID] @@ -350,7 +349,7 @@ var _ = framework.SerialDescribe("[group:underlay]", func() { vpcClient.DeleteSync(vpcName) ginkgo.By("Deleting vlan " + vlanName) - vlanClient.Delete(vlanName, metav1.DeleteOptions{}) + vlanClient.Delete(vlanName) ginkgo.By("Deleting provider network " + providerNetworkName) providerNetworkClient.DeleteSync(providerNetworkName) diff --git a/test/e2e/ovn-vpc-nat-gw/e2e_test.go b/test/e2e/ovn-vpc-nat-gw/e2e_test.go index 4fed16067e9..bd7c22b095a 100644 --- a/test/e2e/ovn-vpc-nat-gw/e2e_test.go +++ b/test/e2e/ovn-vpc-nat-gw/e2e_test.go @@ -427,9 +427,9 @@ var _ = framework.Describe("[group:ovn-vpc-nat-gw]", func() { subnetClient.DeleteSync(underlayExtraSubnetName) ginkgo.By("Deleting vlan " + vlanName) - vlanClient.Delete(vlanName, metav1.DeleteOptions{}) + vlanClient.Delete(vlanName) ginkgo.By("Deleting extra vlan " + vlanExtraName) - vlanClient.Delete(vlanExtraName, metav1.DeleteOptions{}) + vlanClient.Delete(vlanExtraName) ginkgo.By("Deleting provider network " + providerNetworkName) providerNetworkClient.DeleteSync(providerNetworkName) diff --git a/test/e2e/vpc-egress-gateway/e2e_test.go b/test/e2e/vpc-egress-gateway/e2e_test.go new file mode 100644 index 00000000000..85ac4567878 --- /dev/null +++ b/test/e2e/vpc-egress-gateway/e2e_test.go @@ -0,0 +1,322 @@ +package multus + +import ( + "context" + "flag" + "fmt" + "slices" + "strconv" + "strings" + "testing" + "time" + + dockernetwork "github.com/docker/docker/api/types/network" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/component-base/logs" + "k8s.io/klog/v2" + commontest "k8s.io/kubernetes/test/e2e/common" + k8sframework "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/config" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + + apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/util" + "github.com/kubeovn/kube-ovn/test/e2e/framework" + "github.com/kubeovn/kube-ovn/test/e2e/framework/docker" + "github.com/kubeovn/kube-ovn/test/e2e/framework/iproute" + "github.com/kubeovn/kube-ovn/test/e2e/framework/kind" +) + +func init() { + klog.SetOutput(ginkgo.GinkgoWriter) + + // Register flags. + config.CopyFlags(config.Flags, flag.CommandLine) + k8sframework.RegisterCommonFlags(flag.CommandLine) + k8sframework.RegisterClusterFlags(flag.CommandLine) +} + +func TestE2E(t *testing.T) { + k8sframework.AfterReadingAllFlags(&k8sframework.TestContext) + + logs.InitLogs() + defer logs.FlushLogs() + klog.EnableContextualLogging(true) + + gomega.RegisterFailHandler(k8sframework.Fail) + + // Run tests through the Ginkgo runner with output to console + JUnit for Jenkins + suiteConfig, reporterConfig := k8sframework.CreateGinkgoConfig() + klog.Infof("Starting e2e run %q on Ginkgo node %d", k8sframework.RunID, suiteConfig.ParallelProcess) + ginkgo.RunSpecs(t, "Kube-OVN e2e suite", suiteConfig, reporterConfig) +} + +const ( + kindNetwork = "kind" + + controlPlaneLabel = "node-role.kubernetes.io/control-plane" +) + +var clusterName string + +var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { + // Reference common test to make the import valid. + commontest.CurrentSuite = commontest.E2E + + cs, err := k8sframework.LoadClientset() + framework.ExpectNoError(err) + + ginkgo.By("Getting k8s nodes") + k8sNodes, err := e2enode.GetReadySchedulableNodes(context.Background(), cs) + framework.ExpectNoError(err) + + var ok bool + if clusterName, ok = kind.IsKindProvided(k8sNodes.Items[0].Spec.ProviderID); !ok { + ginkgo.Fail("vpc-egress-gateway spec only runs on kind clusters") + } + + return []byte(clusterName) +}, func(data []byte) { + clusterName = string(data) +}) + +var _ = framework.Describe("[group:veg]", func() { + f := framework.NewDefaultFramework("veg") + + var subnetName string + var vpcClient *framework.VpcClient + var subnetClient *framework.SubnetClient + var nadClient *framework.NetworkAttachmentDefinitionClient + var nadName, namespaceName string + var nodes, schedulableNodes []corev1.Node + ginkgo.BeforeEach(func() { + namespaceName = f.Namespace.Name + nadName = "nad-" + framework.RandomSuffix() + subnetName = "subnet-" + framework.RandomSuffix() + vpcClient = f.VpcClient() + subnetClient = f.SubnetClient() + nadClient = f.NetworkAttachmentDefinitionClient() + + nodeList, err := e2enode.GetReadyNodesIncludingTainted(context.Background(), f.ClientSet) + framework.ExpectNoError(err) + framework.ExpectNotEmpty(nodeList.Items) + nodes = nodeList.Items + + nodeList, err = e2enode.GetReadySchedulableNodes(context.Background(), f.ClientSet) + framework.ExpectNoError(err) + framework.ExpectNotEmpty(nodeList.Items) + schedulableNodes = nodeList.Items + }) + ginkgo.AfterEach(func() { + ginkgo.By("Deleting subnet " + subnetName) + subnetClient.DeleteSync(subnetName) + + ginkgo.By("Deleting network attachment definition " + nadName) + nadClient.Delete(nadName) + }) + + framework.ConformanceIt("should be able to create vpc-egress-gateway with underlay subnet", func() { + provider := fmt.Sprintf("%s.%s.%s", nadName, namespaceName, util.OvnProvider) + + ginkgo.By("Creating network attachment definition " + nadName) + nad := framework.MakeOVNNetworkAttachmentDefinition(nadName, namespaceName, provider, nil) + nad = nadClient.Create(nad) + framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config) + + dockerNetworkName := "net-" + framework.RandomSuffix() + ginkgo.By("Creating docker network " + dockerNetworkName) + dockerNetwork, err := docker.NetworkCreate(dockerNetworkName, true, true) + framework.ExpectNoError(err, "creating docker network "+dockerNetworkName) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting docker network " + dockerNetworkName) + err = docker.NetworkRemove(dockerNetworkName) + framework.ExpectNoError(err, "removing docker network "+dockerNetworkName) + }) + + ginkgo.By("Getting kind nodes") + kindNodes, err := kind.ListNodes(clusterName, "") + framework.ExpectNoError(err, "getting nodes in kind cluster") + framework.ExpectNotEmpty(nodes) + + ginkgo.By("Connecting nodes to the docker network") + err = kind.NetworkConnect(dockerNetwork.ID, kindNodes) + framework.ExpectNoError(err, "connecting nodes to network "+dockerNetworkName) + ginkgo.DeferCleanup(func() { + err = kind.NetworkDisconnect(dockerNetwork.ID, kindNodes) + framework.ExpectNoError(err, "disconnecting nodes from network "+dockerNetworkName) + }) + + ginkgo.By("Getting node links that belong to the docker network") + kindNodes, err = kind.ListNodes(clusterName, "") + framework.ExpectNoError(err, "getting nodes in kind cluster") + linkMap := make(map[string]*iproute.Link, len(nodes)) + for _, node := range kindNodes { + links, err := node.ListLinks() + framework.ExpectNoError(err, "failed to list links on node %s: %v", node.Name(), err) + + for _, link := range links { + if link.Address == node.NetworkSettings.Networks[dockerNetworkName].MacAddress { + linkMap[node.Name()] = &link + break + } + } + framework.ExpectHaveKey(linkMap, node.Name()) + } + + providerNetworkName := "pn-" + framework.RandomSuffix() + ginkgo.By("Creating provider network " + providerNetworkName) + var defaultInterface string + customInterfaces := make(map[string][]string, 0) + for node, link := range linkMap { + if defaultInterface == "" { + defaultInterface = link.IfName + } else if link.IfName != defaultInterface { + customInterfaces[link.IfName] = append(customInterfaces[link.IfName], node) + } + } + pn := framework.MakeProviderNetwork(providerNetworkName, false, defaultInterface, customInterfaces, nil) + providerNetworkClient := f.ProviderNetworkClient() + _ = providerNetworkClient.CreateSync(pn) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting provider network " + providerNetworkName) + providerNetworkClient.DeleteSync(providerNetworkName) + }) + + vlanName := "vlan-" + framework.RandomSuffix() + ginkgo.By("Creating vlan " + vlanName) + vlanClient := f.VlanClient() + vlan := framework.MakeVlan(vlanName, providerNetworkName, 0) + _ = vlanClient.Create(vlan) + ginkgo.DeferCleanup(func() { + ginkgo.By("Deleting vlan " + vlanName) + vlanClient.Delete(vlanName) + }) + + ginkgo.By("Getting docker network " + dockerNetworkName) + network, err := docker.NetworkInspect(dockerNetworkName) + framework.ExpectNoError(err, "getting docker network "+dockerNetworkName) + + ginkgo.By("Creating subnet " + subnetName) + subnet := generateSubnetFromDockerNetwork(subnetName, network, f.HasIPv4(), f.HasIPv6()) + subnet.Spec.Provider = provider + + ginkgo.By("Creating underlay subnet " + subnetName) + _ = subnetClient.CreateSync(subnet) + + vpcName := "ovn-cluster" + cidr := framework.RandomCIDR(f.ClusterIPFamily) + bfdIP := framework.RandomIPs(cidr, ";", 1) + ginkgo.By("Enabling BFD Port with IP " + bfdIP + " for VPC " + vpcName) + vpc := vpcClient.Get(vpcName) + patchedVpc := vpc.DeepCopy() + patchedVpc.Spec.BFDPort = &apiv1.BFDPort{ + Enabled: true, + IP: bfdIP, + NodeSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + controlPlaneLabel: "", + }, + }, + } + updatedVpc := vpcClient.PatchSync(vpc, patchedVpc, 3*time.Second) + ginkgo.DeferCleanup(func() { + ginkgo.By("Disabling BFD Port for VPC " + vpcName) + patchedVpc := updatedVpc.DeepCopy() + patchedVpc.Spec.BFDPort.Enabled = false + updatedVpc := vpcClient.PatchSync(updatedVpc, patchedVpc, 3*time.Second) + framework.ExpectFalse(updatedVpc.Status.BFDPort.Enabled) + framework.ExpectEmpty(updatedVpc.Status.BFDPort.Name) + framework.ExpectEmpty(updatedVpc.Status.BFDPort.Nodes) + }) + + framework.ExpectTrue(updatedVpc.Status.BFDPort.Enabled) + framework.ExpectNotEmpty(updatedVpc.Status.BFDPort.Name) + for _, node := range nodes { + if slices.Contains(updatedVpc.Status.BFDPort.Nodes, node.Name) { + framework.ExpectHaveKey(node.Labels, controlPlaneLabel) + } else { + framework.ExpectNotHaveKey(node.Labels, controlPlaneLabel) + } + } + + // TODO: check ovn LRP + + // create vpc egress gateway + replicas := min(len(schedulableNodes), 3) + ginkgo.By("Creating vpc egress gateway with " + strconv.Itoa(replicas) + " replicas") + }) + + framework.ConformanceIt("should be able to create vpc-egress-gateway with macvlan", func() { + provider := fmt.Sprintf("%s.%s", nadName, namespaceName) + + ginkgo.By("Creating network attachment definition " + nadName) + nad := framework.MakeMacvlanNetworkAttachmentDefinition(nadName, namespaceName, "eth0", "bridge", provider, nil) + nad = nadClient.Create(nad) + framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config) + + ginkgo.By("Getting docker network " + kindNetwork) + network, err := docker.NetworkInspect(kindNetwork) + framework.ExpectNoError(err, "getting docker network "+kindNetwork) + + subnet := generateSubnetFromDockerNetwork(subnetName, network, f.HasIPv4(), f.HasIPv6()) + subnet.Spec.Provider = provider + + ginkgo.By("Creating subnet " + subnetName) + _ = subnetClient.CreateSync(subnet) + }) +}) + +func generateSubnetFromDockerNetwork(subnetName string, network *dockernetwork.Inspect, ipv4, ipv6 bool) *apiv1.Subnet { + ginkgo.GinkgoHelper() + + ginkgo.By("Generating subnet configuration from docker network " + network.Name) + var cidrV4, cidrV6, gatewayV4, gatewayV6 string + for _, config := range network.IPAM.Config { + switch util.CheckProtocol(config.Subnet) { + case apiv1.ProtocolIPv4: + if ipv4 { + cidrV4 = config.Subnet + gatewayV4 = config.Gateway + } + case apiv1.ProtocolIPv6: + if ipv6 { + cidrV6 = config.Subnet + if gatewayV6 = config.Gateway; gatewayV6 == "" { + var err error + gatewayV6, err = util.FirstIP(cidrV6) + framework.ExpectNoError(err) + } + } + } + } + cidr := make([]string, 0, 2) + gateway := make([]string, 0, 2) + if ipv4 { + cidr = append(cidr, cidrV4) + gateway = append(gateway, gatewayV4) + } + if ipv6 { + cidr = append(cidr, cidrV6) + gateway = append(gateway, gatewayV6) + } + + excludeIPs := make([]string, 0, len(network.Containers)*2) + for _, container := range network.Containers { + framework.Logf("container %s: %q, %q", container.Name, container.IPv4Address, container.IPv6Address) + if container.IPv4Address != "" && ipv4 { + excludeIPs = append(excludeIPs, strings.Split(container.IPv4Address, "/")[0]) + } + if container.IPv6Address != "" && ipv6 { + excludeIPs = append(excludeIPs, strings.Split(container.IPv6Address, "/")[0]) + } + framework.Logf("excludeIPs: %v", excludeIPs) + } + + framework.Logf("cidr: %v, gateway: %v, excludeIPs: %v", cidr, gateway, excludeIPs) + + return framework.MakeSubnet(subnetName, "", strings.Join(cidr, ","), strings.Join(gateway, ","), "", "", excludeIPs, nil, nil) +} diff --git a/yamls/audit-policy.yaml b/yamls/audit-policy.yaml index 1f196f593c8..8576c6f8190 100644 --- a/yamls/audit-policy.yaml +++ b/yamls/audit-policy.yaml @@ -39,6 +39,8 @@ rules: - vpcs/status - vpc-nat-gateways - vpc-nat-gateways/status + - vpc-egress-gateways + - vpc-egress-gateways/status - vpc-dnses - vpc-dnses/status - subnets