diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3e2215f..094ccd8 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -22,9 +22,7 @@ jobs: - name: Check code run: | ansible-lint -s -x yaml[line-length],var-naming[no-role-prefix] --exclude .github - ansible localhost -m ansible.builtin.template -a "src=edge/roles/edge_install/templates/relocatable_ip.sh.j2 dest=relocatable_ip.sh" -e "{'relocatable_ipv4_subnet': '192.168.7.0/24', 'relocatable_ipv6_subnet': 'fd04::/64', 'edgeCluster':{'relocatable': {'interface': 'eno1'}},'relocatable_interface_macs':'addresses_ipv4[\"11:22:33:44:55:66\"]=\"192.168.7.4/24\";addresses_ipv6[\"11:22:33:44:55:66\"]=\"fd04::4/64\"','cluster_ipv4':true,'cluster_ipv6':true,'controlPlane':{'replicas':3}}" - ansible localhost -m ansible.builtin.template -a "src=edge/roles/edge_install/templates/relocatable_ip.sh.j2 dest=relocatable_ip_sno.sh" -e "{'relocatable_ipv4_subnet': '192.168.7.0/24', 'relocatable_ipv6_subnet': 'fd04::/64', 'edgeCluster':{'relocatable': {'interface': 'eno1'}},'relocatable_interface_macs':'addresses_ipv4[\"11:22:33:44:55:66\"]=\"192.168.7.4/24\";addresses_ipv6[\"11:22:33:44:55:66\"]=\"fd04::4/64\"','cluster_ipv4':true,'cluster_ipv6':true,'controlPlane':{'replicas':1}}" - shellcheck -o all relocatable_ip.sh relocatable_ip_sno.sh edge/roles/edge_csr_approver/files/csr_approver.sh + shellcheck -o all edge/roles/edge_csr_approver/files/csr_approver.sh test-container: runs-on: ubuntu-latest diff --git a/common/roles/install_deps/tasks/main.yaml b/common/roles/install_deps/tasks/main.yaml index f460850..4b49fed 100644 --- a/common/roles/install_deps/tasks/main.yaml +++ b/common/roles/install_deps/tasks/main.yaml @@ -3,3 +3,4 @@ name: - kubernetes - netaddr + - jmespath diff --git a/edge/docs/RELOCATABLE.md b/edge/docs/RELOCATABLE.md index 0c5bfaf..1f63683 100644 --- a/edge/docs/RELOCATABLE.md +++ b/edge/docs/RELOCATABLE.md @@ -1,10 +1,10 @@ # Relocatable Edge Cluster -This feature requires OpenShift 4.12 or higher. +This feature requires OpenShift 4.14 or higher. It also requires MCE 2.5+ (ACM 2.10+). When the ```relocatable``` option is enabled, the cluster is configured in such a way that its primary interface IP addresses can be changed without impacting the operation of the cluster. ## How it works -You set the value of ```relocatable.interface``` to the name of the external facing interface. A secondary static IP address is assigned to this interface. The machineNetwork CIDR is also set to a static internal subnet. Finally, a MachineConfig is created that modifies /etc/default/nodeip-configuration to tell the cluster to use the static IP as the node IP. 'routingViaHost' is also enabled for OCP 4.12. +You set the value of ```relocatable.interface``` to the name of the external facing interface. A secondary static IP address is assigned to this interface. The machineNetwork CIDR is also set to a static internal subnet. Finally, a MachineConfig is created that modifies /etc/default/nodeip-configuration to tell the cluster to use the static IP as the node IP. All of these actions together cause the server to use the static IP for everything related to OpenShift, while still allowing access to the cluster from outside via the primary interface IP address. This means that the external IP can be changed, and the cluster will continue to use the static IP internally for its operation. diff --git a/edge/roles/edge_install/tasks/get_relocatable_ip.yaml b/edge/roles/edge_install/tasks/get_relocatable_ip.yaml deleted file mode 100644 index 74af878..0000000 --- a/edge/roles/edge_install/tasks/get_relocatable_ip.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# in order to make sure that 2 nodes don't use the same IP, we make a list of MAC addresses and assign each MAC an IP -# this is then included in the bash script that gets placed on each node using a MachineConfig - -- name: Increment relocatable IP address - ansible.builtin.set_fact: - interface_number: "{{ (interface_number | int) + 1 }}" - -- name: Create entry for IPv4 address list - when: cluster_ipv4 - block: - - name: Create IPv4 address line - ansible.builtin.set_fact: - bash_item: "addresses_ipv4['{{ item['mac-address'] }}']='{{ relocatable_ipv4_subnet | ansible.utils.nthhost(interface_number | int) }}/{{ relocatable_ipv4_subnet | ansible.utils.ipaddr('prefix') }}'\n" - - - name: Append IPv4 to address list - ansible.builtin.set_fact: - relocatable_interface_macs: '{{ (relocatable_interface_macs | default("")) + bash_item }}' - -- name: Create entry for IPv6 address list - when: cluster_ipv6 - block: - - name: Create IPv6 address line - ansible.builtin.set_fact: - bash_item: "addresses_ipv6['{{ item['mac-address'] }}']='{{ relocatable_ipv6_subnet | ansible.utils.nthhost(interface_number | int) }}/{{ relocatable_ipv6_subnet | ansible.utils.ipaddr('prefix') }}'\n" - - - name: Append IPv6 to address list - ansible.builtin.set_fact: - relocatable_interface_macs: '{{ (relocatable_interface_macs | default("")) + bash_item }}' diff --git a/edge/roles/edge_install/tasks/setup_host_networking.yaml b/edge/roles/edge_install/tasks/setup_host_networking.yaml index 9ea5390..c599783 100644 --- a/edge/roles/edge_install/tasks/setup_host_networking.yaml +++ b/edge/roles/edge_install/tasks/setup_host_networking.yaml @@ -1,3 +1,56 @@ +- name: Calculate relocatable IP + when: edgeCluster.relocatable is defined + block: + - name: Increment relocatable IP address + ansible.builtin.set_fact: + interface_number: "{{ (interface_number | int) + 1 }}" + + - name: Filter the relocatable interface + ansible.builtin.set_fact: + filtered_interface: "{{ nmstate_host.networkConfig.interfaces | json_query('[?name == `' + edgeCluster.relocatable.interface + '`]') | first }}" + + - name: Create entry for IPv4 address list + when: cluster_ipv4 + block: + - name: Get IPv4 addresses + ansible.builtin.set_fact: + ipv4_addresses: "{{ filtered_interface.ipv4.address | default([]) }}" + + - name: Append IPv4 relocatable address + ansible.builtin.set_fact: + ipv4_addresses: "{{ ipv4_addresses + [{'ip': relocatable_ipv4_subnet | ansible.utils.nthhost(interface_number | int), 'prefix-length': relocatable_ipv4_subnet | ansible.utils.ipaddr('prefix')}] }}" + + - name: Update IPv4 addresses + ansible.builtin.set_fact: + filtered_interface: "{{ filtered_interface | combine({'ipv4': {'address': ipv4_addresses}}, recursive=true) }}" + + - name: Create entry for IPv6 address list + when: cluster_ipv6 + block: + - name: Get IPv6 addresses + ansible.builtin.set_fact: + ipv6_addresses: "{{ filtered_interface.ipv6.address | default([]) }}" + + - name: Append IPv6 relocatable address + ansible.builtin.set_fact: + ipv6_addresses: "{{ ipv6_addresses + [{'ip': relocatable_ipv6_subnet | ansible.utils.nthhost(interface_number | int), 'prefix-length': relocatable_ipv6_subnet | ansible.utils.ipaddr('prefix')}] }}" + + - name: Update IPv6 addresses + ansible.builtin.set_fact: + filtered_interface: "{{ filtered_interface | combine({'ipv6': {'address': ipv6_addresses}}, recursive=true) }}" + + - name: Get all non-reloctable interfaces + ansible.builtin.set_fact: + interfaces_list: "{{ nmstate_host.networkConfig.interfaces | json_query('[?name != `' + edgeCluster.relocatable.interface + '`]') }}" + + - name: Add reloctable interface + ansible.builtin.set_fact: + interfaces_list: "{{ interfaces_list + [filtered_interface] }}" + + - name: Create new networkConfig + ansible.builtin.set_fact: + relocatable_network_config: "{{ nmstate_host.networkConfig | combine({'interfaces': interfaces_list}, recursive=true) }}" + - name: Create NMStateConfig kubernetes.core.k8s: template: NMStateConfig.yaml.j2 @@ -5,13 +58,3 @@ state: present register: k8s_result until: k8s_result is not failed - -- name: Calculate relocatable IP - when: edgeCluster.relocatable is defined - block: - - name: Get relocatable interface index - loop: "{{ nmstate_host.networkConfig.interfaces }}" - loop_control: - label: "{{ item.name }}" - when: item.name == edgeCluster.relocatable.interface - ansible.builtin.include_tasks: get_relocatable_ip.yaml diff --git a/edge/roles/edge_install/templates/InfraEnv.yaml.j2 b/edge/roles/edge_install/templates/InfraEnv.yaml.j2 index 356298d..2fbccc3 100644 --- a/edge/roles/edge_install/templates/InfraEnv.yaml.j2 +++ b/edge/roles/edge_install/templates/InfraEnv.yaml.j2 @@ -21,6 +21,3 @@ spec: proxy: {{ proxy | to_nice_yaml(indent=2) | trim | indent(4) }} {% endif %} -{% if edgeCluster.relocatable is defined %} - ignitionConfigOverride: '{{ (lookup('ansible.builtin.template', 'RelocatableConfig.yaml.j2') | from_yaml | to_json) }}' -{% endif %} diff --git a/edge/roles/edge_install/templates/NMStateConfig.yaml.j2 b/edge/roles/edge_install/templates/NMStateConfig.yaml.j2 index f077a73..8b3bbeb 100644 --- a/edge/roles/edge_install/templates/NMStateConfig.yaml.j2 +++ b/edge/roles/edge_install/templates/NMStateConfig.yaml.j2 @@ -7,7 +7,11 @@ metadata: nmstate-cluster: "{{ metadata.name }}" spec: config: +{% if edgeCluster.relocatable is defined %} + {{ relocatable_network_config | to_nice_yaml(indent=2) | trim | indent(4) }} +{% else %} {{ nmstate_host.networkConfig | to_nice_yaml(indent=2) | trim | indent(4) }} +{% endif %} interfaces: {% for interface in nmstate_host.networkConfig.interfaces %} {% if interface.type == "ethernet" %} diff --git a/edge/roles/edge_install/templates/RelocatableConfig.yaml.j2 b/edge/roles/edge_install/templates/RelocatableConfig.yaml.j2 deleted file mode 100644 index e060c3d..0000000 --- a/edge/roles/edge_install/templates/RelocatableConfig.yaml.j2 +++ /dev/null @@ -1,28 +0,0 @@ -ignition: - version: 3.2.0 -storage: - files: - - contents: - source: data:text/plain;charset=utf-8;base64,{{ lookup('ansible.builtin.template', 'relocatable_ip.sh.j2') | b64encode }} - mode: 0700 - overwrite: true - path: /var/local/relocatable/relocatable_ip.sh - user: - name: root -systemd: - units: - - contents: | - [Unit] - Description=Add relocatable IP - After=network.target ovs-configuration.service - Before=network-online.target kubelet.service crio.service - - [Service] - User=root - Type=oneshot - ExecStart=/bin/bash -c /var/local/relocatable/relocatable_ip.sh - - [Install] - WantedBy=network-online.target - enabled: true - name: relocatable-ip.service diff --git a/edge/roles/edge_install/templates/RelocatableConfigMap.yaml.j2 b/edge/roles/edge_install/templates/RelocatableConfigMap.yaml.j2 index 9ab168f..cb49df1 100644 --- a/edge/roles/edge_install/templates/RelocatableConfigMap.yaml.j2 +++ b/edge/roles/edge_install/templates/RelocatableConfigMap.yaml.j2 @@ -26,14 +26,4 @@ data: path: /etc/default/nodeip-configuration user: name: root - relocatable_ip_{{ node_type }}.yaml: | - apiVersion: machineconfiguration.openshift.io/v1 - kind: MachineConfig - metadata: - labels: - machineconfiguration.openshift.io/role: {{ node_type }} - name: 99-{{ node_type }}-relocatable-ip - spec: - config: - {{ lookup('ansible.builtin.template', 'RelocatableConfig.yaml.j2') | indent(8) }} {% endfor %} diff --git a/edge/roles/edge_install/templates/RoutingViaHost.yaml.j2 b/edge/roles/edge_install/templates/RoutingViaHost.yaml.j2 index 782bbbf..ce2adf7 100644 --- a/edge/roles/edge_install/templates/RoutingViaHost.yaml.j2 +++ b/edge/roles/edge_install/templates/RoutingViaHost.yaml.j2 @@ -4,8 +4,7 @@ metadata: name: "routing-via-host" namespace: "{{ metadata.name }}" data: -{% if edgeCluster.routingViaHost | default(false) or - (edgeCluster.relocatable is defined and cluster_deployment.resources[0].status.installVersion is ansible.builtin.version("4.13.0", "lt")) %} +{% if edgeCluster.routingViaHost | default(false) %} routing_via_host.yaml: | apiVersion: operator.openshift.io/v1 kind: Network diff --git a/edge/roles/edge_install/templates/relocatable_ip.sh.j2 b/edge/roles/edge_install/templates/relocatable_ip.sh.j2 deleted file mode 100644 index 7955493..0000000 --- a/edge/roles/edge_install/templates/relocatable_ip.sh.j2 +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash - -# we are required to add the relocatable IP here, rather than in the NMStateConfig -# -# this is because NMState ignores static IP addresses when DHCP is enabled -# see https://github.com/nmstate/nmstate/blob/a323ad9d1ae377ae39971f0f0a9f39d3ad341765/rust/src/lib/ip.rs#L190-L197 -# this appears to have been fixed in NMState v2.2.10 (https://github.com/nmstate/nmstate/pull/2303), but I'm not sure when this will make it into OCP/ACM -# -# configure-ovs.sh makes this same assumption, and wipes out any static IPs when DHCP is in use -# see https://github.com/openshift/machine-config-operator/blob/master/templates/common/_base/files/configure-ovs-network.yaml - -mac_address=$(cat /sys/class/net/{{ edgeCluster.relocatable.interface }}/address) - -declare -A addresses_ipv4 -declare -A addresses_ipv6 - -{{ relocatable_interface_macs }} - -for interface in {{ edgeCluster.relocatable.interface }} ovs-if-br-ex; do - if ! nmcli -g connection.id connection show "${interface}"; then - echo "---" - echo "interface ${interface} does not exist, skipping" - echo "---" - continue - fi - dev_name=$(nmcli -g connection.interface-name connection show "${interface}") -{% if cluster_ipv4 %} - current_ipv4_addresses=$(nmcli -g ipv4.addresses connection show "${interface}") - if [[ "${current_ipv4_addresses}" != *"${addresses_ipv4[${mac_address}]}"* ]]; then - nmcli connection modify "${interface}" +ipv4.addresses "${addresses_ipv4[${mac_address}]}"{% if controlPlane.replicas == 3 %} +ipv4.dns "{{ relocatable_ipv4_subnet | ansible.utils.nthhost(2) }}"{% endif +%} - nmcli dev reapply "${dev_name}" - echo "---" - echo "added relocatable IPv4 to connection ${interface}, device ${dev_name}" - echo "---" - else - echo "---" - echo "relocatable IPv4 already attached to connection ${interface}, device ${dev_name}" - echo "---" - fi -{% endif %} - -{% if cluster_ipv6 %} - current_ipv6_addresses=$(nmcli -e no -g ipv6.addresses connection show "${interface}") - if [[ "${current_ipv6_addresses}" != *"${addresses_ipv6[${mac_address}]}"* ]]; then - nmcli connection modify "${interface}" +ipv6.addresses "${addresses_ipv6[${mac_address}]}"{% if controlPlane.replicas == 3 %} +ipv6.dns "{{ relocatable_ipv6_subnet | ansible.utils.nthhost(2) }}"{% endif +%} - nmcli dev reapply "${dev_name}" - echo "---" - echo "added relocatable IPv6 to connection ${interface}, device ${dev_name}" - echo "---" - else - echo "---" - echo "relocatable IPv6 already attached to connection ${interface}, device ${dev_name}" - echo "---" - fi -{% endif %} -done diff --git a/edge/roles/edge_post_install/defaults/main.yaml b/edge/roles/edge_post_install/defaults/main.yaml index 4d4e628..1212a36 100644 --- a/edge/roles/edge_post_install/defaults/main.yaml +++ b/edge/roles/edge_post_install/defaults/main.yaml @@ -1 +1,3 @@ access_control_node: false +node_ipv4_addresses: [] +node_ipv6_addresses: [] diff --git a/edge/roles/edge_post_install/tasks/main.yaml b/edge/roles/edge_post_install/tasks/main.yaml index 707184f..5e25b2a 100644 --- a/edge/roles/edge_post_install/tasks/main.yaml +++ b/edge/roles/edge_post_install/tasks/main.yaml @@ -39,44 +39,49 @@ ansible.builtin.set_fact: access_control_node: true - - name: Get BareMetalHosts + - name: Get Agents kubernetes.core.k8s_info: - api_version: metal3.io/v1alpha1 - kind: BareMetalHost + api_version: agent-install.openshift.io/v1beta1 + kind: Agent namespace: "{{ metadata.name }}" - register: bmh_list - until: bmh_list is not failed + register: agent_list + until: agent_list is not failed - name: Find control plane node - loop: "{{ bmh_list.resources }}" + loop: "{{ agent_list.resources }}" loop_control: label: "{{ item.metadata.name }}" - when: item.metadata.annotations['bmac.agent-install.openshift.io/role'] == "master" + when: item.spec.role == "master" ansible.builtin.set_fact: control_plane_node: "{{ item }}" - - name: Find IPv4 for first control plane node - loop: "{{ control_plane_node.status.hardware.nics }}" + - name: Get list of IPv4 addresses for first control plane node + loop: "{{ control_plane_node.status.inventory.interfaces }}" loop_control: - label: "{{ item.ip | default(item.name) }}" + label: "{{ item.name }}" + ansible.builtin.set_fact: + node_ipv4_addresses: "{{ node_ipv4_addresses + item.ipV4Addresses }}" + + - name: Find suitable IPv4 address for first control plane node + loop: "{{ node_ipv4_addresses }}" when: - - item.ip is defined - - item.ip | ansible.utils.ipv4 - - not (relocatable_ipv4_subnet | ansible.utils.network_in_usable(item.ip)) + - not (relocatable_ipv4_subnet | ansible.utils.network_in_usable(item | ansible.utils.ipaddr('address'))) ansible.builtin.set_fact: - node_ip: "{{ item.ip }}" + node_ip: "{{ item | ansible.utils.ipaddr('address') }}" - - name: Find IPv6 for first control plane node - loop: "{{ control_plane_node.status.hardware.nics }}" + - name: Get list of IPv6 addresses for first control plane node + loop: "{{ control_plane_node.status.inventory.interfaces }}" loop_control: - label: "{{ item.ip | default(item.name) }}" + label: "{{ item.name }}" + ansible.builtin.set_fact: + node_ipv6_addresses: "{{ node_ipv6_addresses + item.ipV6Addresses }}" + + - name: Find suitable IPv6 address for first control plane node + loop: "{{ node_ipv6_addresses }}" when: - - node_ip is not defined - - item.ip is defined - - item.ip | ansible.utils.ipv6 - - not (relocatable_ipv6_subnet | ansible.utils.network_in_usable(item.ip)) + - not (relocatable_ipv6_subnet | ansible.utils.network_in_usable(item | ansible.utils.ipaddr('address'))) ansible.builtin.set_fact: - node_ip: "{{ item.ip }}" + node_ip: "{{ item | ansible.utils.ipaddr('address') }}" - name: Add required environment vars ansible.builtin.set_fact: