From 7dc4a62b7b8f986f27198d675f26c3444c893986 Mon Sep 17 00:00:00 2001
From: William Graef <william.graef@oracle.com>
Date: Tue, 9 Jul 2024 17:12:34 -0400
Subject: [PATCH] add ocne vlan deployment

---
 ocne/default_vars.yml     |   1 +
 ocne/deploy_ocne_vlan.yml | 363 ++++++++++++++++++++++++++++++++++++++
 ocne/host_setup.yml       |   3 +-
 3 files changed, 365 insertions(+), 2 deletions(-)
 create mode 100644 ocne/deploy_ocne_vlan.yml

diff --git a/ocne/default_vars.yml b/ocne/default_vars.yml
index e366639..b396d14 100644
--- a/ocne/default_vars.yml
+++ b/ocne/default_vars.yml
@@ -26,6 +26,7 @@ debug_enabled: false
 ocne_type: quick
 use_ocne_full: false
 use_lb: false
+use_int_lb: false
 oci_ccm_bash: false
 use_oci_ccm: false
 use_istio: false
diff --git a/ocne/deploy_ocne_vlan.yml b/ocne/deploy_ocne_vlan.yml
new file mode 100644
index 0000000..205f9c5
--- /dev/null
+++ b/ocne/deploy_ocne_vlan.yml
@@ -0,0 +1,363 @@
+---
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# This software is made available to you under the terms of the Universal Permissive License (UPL), Version 1.0.
+# The Universal Permissive License (UPL), Version 1.0 (see COPYING or https://oss.oracle.com/licenses/upl)
+# See LICENSE.TXT for details.
+
+- name: OCNE OS pre-installation steps
+  hosts: operator,controlplane,worker
+  become: true
+  vars_files:
+    - default_vars.yml
+    - oci_vars.yml
+
+  tasks:
+
+  # SELinux for iptables
+  # https://bugzilla.redhat.com/show_bug.cgi?id=2008097
+
+    - name: Write cil selinux policy file
+      ansible.builtin.shell: |
+        echo '(allow iptables_t cgroup_t (dir (ioctl)))' > /home/{{ username }}/local_iptables.cil
+      register: result
+      changed_when: result.rc == 0
+
+    - name: Add selinux module
+      ansible.builtin.shell: |
+        semodule -i /home/{{ username }}/local_iptables.cil
+      register: result
+      changed_when: result.rc == 0
+
+    - name: Setup olcne ol8 repos
+      ansible.builtin.include_tasks: ol8_repo_config.yml
+      when: ansible_distribution == 'OracleLinux' and ansible_distribution_major_version == '8'
+
+    - name: Setup olcne ol9 repos
+      ansible.builtin.include_tasks: ol9_repo_config.yml
+      when: ansible_distribution == 'OracleLinux' and ansible_distribution_major_version == '9'
+
+    - name: Install chrony
+      ansible.builtin.dnf:
+        name: chrony
+        state: present
+
+    - name: Enable chrony service
+      ansible.builtin.systemd_service:
+        name: chronyd
+        state: started
+        enabled: true
+
+    - name: Disable swap
+      ansible.builtin.shell: |
+        swapoff -a
+      register: disable_swap
+      changed_when: disable_swap.rc == 0
+
+    - name: Disable swap in fstab
+      ansible.builtin.replace:
+        path: /etc/fstab
+        regexp: '^([^#].*?\sswap\s+sw\s+.*)$'
+        replace: '# \1'
+
+- name: OCNE installation steps
+  hosts: operator
+  become: true
+  vars_files:
+    - default_vars.yml
+    - oci_vars.yml
+
+  vars:
+    operator_nodes: "{{ groups['operator'] | map('extract', hostvars, ['ansible_ens5', 'ipv4', 'address']) | join(',') }}"
+    control_nodes: "{{ groups['controlplane'] | map('extract', hostvars, ['ansible_ens5', 'ipv4', 'address']) | join(',') }}"
+    worker_nodes: "{{ groups['worker'] | map('extract', hostvars, ['ansible_ens5', 'ipv4', 'address']) | join(',') }}"
+    all_nodes: "{{ operator_nodes + ',' + control_nodes + ',' + worker_nodes }}"
+
+  tasks:
+
+    - name: Print OCI compartment_ocid
+      ansible.builtin.debug:
+        msg: OCI compartment OCID is {{ my_compartment_id }}
+      when: debug_enabled
+
+    - name: Print OCI vcn_ocid
+      ansible.builtin.debug:
+        msg: OCI vcn OCID is {{ my_vcn_id }}
+      when: debug_enabled
+
+    - name: Print OCI lb_subnet_ocid
+      ansible.builtin.debug:
+        msg: OCI lb_subnet OCID is {{ my_subnet_id }}
+      when: debug_enabled
+
+    - name: Print vcn subnet_domain_name
+      ansible.builtin.debug:
+        msg: OCI Subnet Domain Name is {{ my_subnet_domain_name }}
+      when:
+        - debug_enabled
+
+    - name: Print string of all operator nodes comma separated
+      ansible.builtin.debug:
+        msg: "{{ operator_nodes }}"
+      when:
+        - debug_enabled
+
+    - name: Print string of all control plane nodes comma separated
+      ansible.builtin.debug:
+        msg: "{{ control_nodes }}"
+      when:
+        - debug_enabled
+
+    - name: Print string of all worker nodes comma separated
+      ansible.builtin.debug:
+        msg: "{{ worker_nodes }}"
+      when:
+        - debug_enabled
+
+    - name: Print string of all nodes comma separated
+      ansible.builtin.debug:
+        msg: "{{ all_nodes }}"
+      when:
+        - debug_enabled
+
+    - name: Add compartment variable to .bashrc
+      ansible.builtin.blockinfile:
+        dest: /home/oracle/.bashrc
+        block: |
+          export COMPARTMENT_OCID={{ my_compartment_id }}
+          export LC_ALL=C.UTF-8
+          export LANG=C.UTF-8
+        insertafter: EOF
+        owner: "{{ username }}"
+
+    - name: Install oci-cli on ol8
+      ansible.builtin.dnf:
+        name: ['python36-oci-cli', 'jq']
+        enablerepo: ol8_developer
+        state: present
+      when:
+        - ansible_facts['distribution'] == "OracleLinux"
+        - ansible_facts['distribution_major_version'] == "8"
+
+    - name: Install oci-cli on ol9
+      ansible.builtin.dnf:
+        name: ['python39-oci-cli', 'jq']
+        state: present
+      when:
+        - ansible_facts['distribution'] == "OracleLinux"
+        - ansible_facts['distribution_major_version'] == "9"
+
+    - name: Add operator firewall rules
+      ansible.posix.firewalld:
+        port: 8091/tcp
+        permanent: true
+        state: enabled
+        immediate: true
+
+    - name: Add control plane firewall rules
+      ansible.posix.firewalld:
+        port: "{{ item[1] }}"
+        permanent: true
+        state: enabled
+        immediate: true
+      delegate_to: "{{ item[0] }}"
+      loop: "{{ groups['controlplane'] | product(['2379/tcp', '2380/tcp', '6443/tcp', '8090/tcp', '8472/udp', '10250/tcp', '10255/tcp', '10251/tcp', '10252/tcp']) | list }}"
+
+    - name: Add firewall rules for internal lb
+      when: use_int_lb
+      block:
+        - name: Add internal lb firewall rule
+          ansible.posix.firewalld:
+            port: "{{ item }}"
+            permanent: true
+            state: enabled
+            immediate: true
+          with_items:
+            - 6444/tcp
+
+        - name: Add vrrp firewall rule
+          ansible.builtin.shell: |
+            firewall-cmd --add-protocol=vrrp --zone=public --permanent
+            firewall-cmd --reload
+          register: vrrp_firewall
+          changed_when: vrrp_firewall.rc == 0
+
+    - name: Add worker firewall rules
+      ansible.posix.firewalld:
+        port: "{{ item[1] }}"
+        permanent: true
+        state: enabled
+        immediate: true
+      delegate_to: "{{ item[0] }}"
+      loop: "{{ groups['worker'] | product(['8090/tcp', '10250/tcp', '10255/tcp', '8472/udp']) | list }}"
+
+    - name: Add cni0 firewall rule
+      ansible.posix.firewalld:
+        zone: trusted
+        interface: cni0
+        permanent: true
+        state: enabled
+        immediate: true
+      delegate_to: "{{ item }}"
+      loop: "{{ (groups['controlplane'] + groups['worker']) }}"
+
+    - name: Load br_netfilter module
+      community.general.modprobe:
+        name: br_netfilter
+        state: present
+      delegate_to: "{{ item }}"
+      loop: "{{ (groups['controlplane'] + groups['worker']) }}"
+
+    - name: Ensure br_netfilter module loads on boot
+      ansible.builtin.shell: |
+        echo "br_netfilter" > /etc/modules-load.d/br_netfilter.conf
+      delegate_to: "{{ item }}"
+      loop: "{{ (groups['controlplane'] + groups['worker']) }}"
+      register: load_netfilter
+      changed_when: load_netfilter.rc == 0
+
+    - name: Disable docker and containerd services
+      ansible.builtin.systemd_service:
+        name: "{{ item[1] }}"
+        state: stopped
+        enabled: false
+      when: ansible_facts.services['docker.service'] is defined or ansible_facts.servcices['containerd.service'] is defined
+      delegate_to: "{{ item[0] }}"
+      loop: "{{ (groups['controlplane'] + groups['worker']) | product(['docker.service', 'containerd.service']) | list }}"
+
+    - name: Install olcne packages on operator
+      ansible.builtin.dnf:
+        name:
+          - olcnectl
+          - olcne-api-server
+          - olcne-utils
+        state: present
+
+    - name: Install olcne packages on control plane and worker
+      ansible.builtin.dnf:
+        name:
+          - olcne-utils
+          - olcne-agent
+        state: present
+      delegate_to: "{{ item }}"
+      loop: "{{ (groups['controlplane'] + groups['worker']) }}"
+
+    - name: Check if OCNE provisioned
+      ansible.builtin.stat:
+        path: ~/.ocne-provisioned
+      become: true
+      become_user: "{{ username }}"
+      register: ocne_provision
+
+    - name: Enable olcne api-server and agent service
+      ansible.builtin.systemd_service:
+        name: olcne-api-server
+        state: stopped
+        enabled: true
+      when: not ocne_provision.stat.exists
+
+    - name: Enable olcne api-server and agent service
+      ansible.builtin.systemd_service:
+        name: olcne-agent
+        state: stopped
+        enabled: true
+      when: not ocne_provision.stat.exists
+      delegate_to: "{{ item }}"
+      loop: "{{ (groups['controlplane'] + groups['worker']) }}"
+
+    - name: Add user to olcne group
+      ansible.builtin.user:
+        name: "{{ username }}"
+        groups: olcne
+        append: true
+
+    - name: Reset ssh connection to allow user changes to affect 'current login user'
+      ansible.builtin.meta: reset_connection
+
+    - name: Check if OCNE provisioned
+      ansible.builtin.stat:
+        path: ~/.ocne-provisioned
+      become: true
+      become_user: "{{ username }}"
+      register: ocne_provision
+
+    - name: Create Certificates for Kubernetes Nodes
+      ansible.builtin.shell: |
+        olcnectl certificates distribute --cert-dir ~/certificates/ --nodes {{ all_nodes }}
+      become: true
+      become_user: "{{ username }}"
+      when: not ocne_provision.stat.exists
+      register: create_certs
+      changed_when: create_certs.rc == 0
+
+    - name: Create Certificates for the Platform CLI
+      ansible.builtin.shell: |
+        olcnectl certificates generate --nodes {{ operator_nodes }},127.0.0.1 \
+        --cert-dir ~/.olcne/certificates/{{ operator_nodes }}:8091/  \
+        --byo-ca-cert ~/certificates/ca/ca.cert \
+        --byo-ca-key ~/certificates/ca/ca.key \
+        --one-cert
+      become: true
+      become_user: "{{ username }}"
+      when: not ocne_provision.stat.exists
+      register: create_certs
+      changed_when: create_certs.rc == 0
+
+    - name: Copy CA Certificate for the Platform CLI
+      ansible.builtin.copy:
+        src: ~/certificates/ca/ca.cert
+        dest: ~/.olcne/certificates/ocne-operator:8091/ca.cert
+        remote_src: true
+        mode: "0644"
+      become: true
+      become_user: "{{ username }}"
+      when: not ocne_provision.stat.exists
+
+    # - name: Workaround for permissions on certificate files due to Ansible
+    #   ansible.builtin.file:
+    #     path: '/etc/olcne/certificates/{{ item }}.cert'
+    #     mode: '0640'
+    #   loop:
+    #     - node
+    #     - ca
+    #   when: not ocne_provision.stat.exists
+
+    - name: Start OLCNE API Server with Certificate
+      ansible.builtin.shell: |
+        /etc/olcne/bootstrap-olcne.sh --secret-manager-type file --olcne-component api-server
+      when: not ocne_provision.stat.exists
+      register: start_api_server
+      changed_when: start_api_server.rc == 0
+
+    - name: Start OLCNE Agent with the Certificates
+      become: true
+      ansible.builtin.shell: |
+        /etc/olcne/bootstrap-olcne.sh --secret-manager-type file --olcne-component agent
+      delegate_to: "{{ item }}"
+      loop: "{{ ((range(groups['controlplane'] | length - empty_cp_nodes | int) | map('extract', groups['controlplane']) | list) + (range(groups['worker'] | length - empty_wrk_nodes | int) | map('extract', groups['worker']) | list)) }}"
+      when: not ocne_provision.stat.exists
+      register: start_agent
+      changed_when: start_agent.rc == 0
+
+    - name: Create Certificates for ExternalIPs service
+      ansible.builtin.shell: |
+        olcnectl certificates generate \
+        --nodes externalip-validation-webhook-service.externalip-validation-system.svc,externalip-validation-webhook-service.externalip-validation-system.svc.cluster.local \
+        --cert-dir /home/{{ username }}/certificates/restrict_external_ip/ \
+        --byo-ca-cert /home/{{ username }}/certificates/ca/ca.cert \
+        --byo-ca-key /home/{{ username }}/certificates/ca/ca.key \
+        --one-cert
+      become: true
+      become_user: "{{ username }}"
+      when: not ocne_provision.stat.exists
+      register: create_ext_certs
+      changed_when: create_ext_certs.rc == 0
+
+    - name: Copy CA Certificate for the ExternalIP service
+      ansible.builtin.copy:
+        src: ~/certificates/ca/ca.cert
+        dest: ~/certificates/restrict_external_ip/ca.cert
+        remote_src: true
+        mode: "0644"
+      become: true
+      become_user: "{{ username }}"
+      when: not ocne_provision.stat.exists
diff --git a/ocne/host_setup.yml b/ocne/host_setup.yml
index c77d367..56ff99b 100644
--- a/ocne/host_setup.yml
+++ b/ocne/host_setup.yml
@@ -125,8 +125,7 @@
         {{ hostvars[item].ansible_hostname }},\
         {{ hostvars[item].ansible_default_ipv4.address }},\
         {{ hostvars[item].ansible_hostname + '.' + my_subnet_domain_name }} >> ~/.ssh/known_hosts
-      with_items:
-        - "{{ groups['all'] }}"
+      loop: "{{ groups['all'] | flatten(levels=1) }}"
       become: true
       become_user: "{{ username }}"
       register: result