Move deployment tests to CI-run and re-enable integration tests

.cirun.yml

@@ -0,0 +1,16 @@
+runners:
+  - name: cirun-aws-runner
+    # Cloud Provider: AWS
+    cloud: aws
+    # Instance Type has 4 vcpu, 16 GiB memory, Up to 5 Gbps Network Performance
+    instance_type: t3.2xlarge
+    machine_image: ami-053053586808c3e70 
+    # ami-0a388df278199ff52
+    # Region: Oregon
+    region: us-west-2
+    # Use Spot Instances for cost savings
+    preemptible:
+      - true
+      - false
+    labels:
+      - cirun-runner

.github/scripts/extract_network_info.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+IFS=$'\n' # Split output into lines based on newline
+
+# Use `ip address show` instead of `ip -br -4 address show` for broader compatibility
+readarray -t lines <<< "$(ip -br -4 address show | grep UP)"
+for line in "${lines[@]}"; do
+  if [[ $line =~ (eth[0-9]|ens[0-9]+|enp[0-9].*) ]]; then
+    INTERFACE=$(echo $line | awk '{print $1}')
+    IP_RANGE=$(echo $line | awk '{print $3}')
+    break
+  fi
+done
+
+# Write variables into network_info.txt
+echo "Interface: $INTERFACE" > network_info.txt
+echo "IP Range: $IP_RANGE" >> network_info.txt

.github/scripts/gen_inventory.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Check if the correct number of arguments was provided
+if [ $# -ne 2 ]; then
+    echo "Usage: $0 <hostname> <output_path>"
+    exit 1
+fi
+
+# Get the hostname from the first argument
+HOSTNAME=$1
+
+# Get the output path from the second argument
+OUTPUT_PATH=$2
+
+# Ensure the directory exists
+mkdir -p $(dirname "$OUTPUT_PATH")
+
+# Create the inventory.ini file at the specified output path with dynamic content
+cat <<EOF > "$OUTPUT_PATH"
+${HOSTNAME} connection=local ansible_ssh_host=127.0.0.1
+
+[hpc_master]
+${HOSTNAME}
+
+[hpc_worker]
+${HOSTNAME}
+EOF
+
+echo "inventory.ini file has been created at $OUTPUT_PATH."

.github/workflows/kvm-test.yaml

@@ -1,41 +1,128 @@
 ---
-name: Vagrant (KVM) Tests
-
-on:
-  pull_request:
-  push:
-    branches:
-      - main
-
-jobs:
-  # https://github.com/jonashackt/vagrant-github-actions
-  test-kvm:
-    name: KVM Test
-    runs-on: macos-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Cache Vagrant boxes
-        uses: actions/cache@v2
-        with:
-          path: ~/.vagrant.d/boxes
-          key: ${{ runner.os }}-vagrant-${{ hashFiles('Vagrantfile') }}
-          restore-keys: |
-            ${{ runner.os }}-vagrant-
-
-      - name: Install test dependencies.
-        run: sudo pip3 install ansible
-
-      - name: Install Ansible Dependencies
-        working-directory: tests/ubuntu2004-singlenode
-        run: |
-          ansible-galaxy collection install community.general
-          ansible-galaxy collection install ansible.posix
-
-      - name: Show Vagrant version
-        run: vagrant --version
-
-# Disabled until we fix it
-#      - name: Run vagrant up
-#        working-directory: tests/ubuntu2004-singlenode
-#        run: vagrant up
+  name: Vagrant (KVM) Tests
+
+  on:
+    pull_request:
+    push:
+      branches:
+        - main
+
+  jobs:
+    test-kvm:
+      name: KVM Test
+      # Disable ci-run untill addressing /var/lib/dpkg/lock-frontend issue
+      runs-on: "cirun-runner--${{ github.run_id }}"
+      # runs-on: "ubuntu-latest"
+      steps:
+        - uses: actions/checkout@v4
+
+        - uses: actions/setup-python@v5
+          with:
+            python-version: "3.10"
+            cache: "pip"
+
+        - name: Install dependencies
+          run: |
+            pip install ansible
+
+        - name: Install Ansible dependencies
+          run: |
+            ansible-galaxy collection install -r requirements.yaml
+
+        - name: Create deploy folder and move inventory files
+          run: |
+            mkdir deploy
+            cp -r inventory.template/* deploy/
+
+            chmod +x .github/scripts/gen_inventory.sh
+            ./.github/scripts/gen_inventory.sh $(hostname -s) deploy/inventory.ini
+
+        - name: Check network adapter
+          run: |
+            ip a
+
+        - name: Check hosts
+          run: |
+            cat /etc/hosts
+
+        - name: Extract Network Information
+          run: |
+            chmod +x .github/scripts/extract_network_info.sh
+            ./.github/scripts/extract_network_info.sh
+            echo "adapter_name=$(cat network_info.txt | head -1 | awk '{print $2}')" >> $GITHUB_ENV
+            echo "ip_range=$(cat network_info.txt | awk 'NR > 1 && $3 {print $3}')" >> $GITHUB_ENV
+
+        - name: Update group vars
+          run: |
+            cp deploy/group_vars/all.yaml deploy/group_vars/all.yaml.bak
+
+            echo "Updating group vars for firewall and internal network"
+            echo "firewall_internal_ip_range: $ip_range" >> deploy/group_vars/all.yaml
+            echo "internal_interface: $adapter_name" >> deploy/group_vars/all.yaml
+            echo "SlurmConfigFileDIr: /etc/slurm" >> deploy/group_vars/all.yaml
+
+            echo "Replace hpc01-test with $(hostname -s) in group_vars/hpc_worker.yaml file"
+            sed -i "s/hpc01-test/$(hostname -s)/g" deploy/group_vars/hpc_worker.yaml
+
+            echo "Replace LDAP server URI with $(hostname -s) in group_vars/all.yaml file"
+            sed -i "s|ldap://hpc01-test:389|ldap://$(hostname -s):389|g" deploy/group_vars/all.yaml
+
+            diff deploy/group_vars/all.yaml.bak deploy/group_vars/all.yaml || true
+
+        - name: Disable unattended-upgrades
+          run: |
+            # Ensure all commands are non-interactive by setting DEBIAN_FRONTEND to noninteractive
+            export DEBIAN_FRONTEND=noninteractive
+
+            # Check if unattended-upgrades service is active and stop it if it is
+            if systemctl is-active --quiet unattended-upgrades; then
+                echo "Stopping unattended-upgrades service..."
+                sudo systemctl stop unattended-upgrades
+            else
+                echo "unattended-upgrades service is not active. Skipping stop command."
+            fi
+
+            # Proceed with killing any running APT processes without manual confirmation
+            echo "Checking and killing running APT processes if necessary..."
+            sudo lsof /var/lib/dpkg/lock-frontend | awk '{print $2}' | tail -n +2 | while read PID; do
+                if [ ! -z "$PID" ]; then
+                    echo "Killing PID $PID"
+                    sudo kill -9 $PID
+                fi
+            done
+
+            # Configure any packages that are in an unclean state non-interactively
+            echo "Configuring any packages in an unclean state..."
+            sudo dpkg --configure -a
+
+            # Remove unattended-upgrades to avoid automatic background updates during script execution
+            echo "Disabling unattended upgrades..."
+            sudo apt-get remove --purge unattended-upgrades -y || true
+
+        # - name:  Run ssh session
+        #   uses: mxschmitt/action-tmate@v3
+        #   with:
+        #     detached: true
+        - name: Move hpc variables from group_vars to temp location
+          run: |
+            mv deploy/group_vars/hpc_*.yaml /tmp
+
+        - name: Run ansible playbook (partial)
+          run: |
+            cd deploy
+            ansible-playbook ../playbook.yaml -i inventory.ini --connection=local -v
+          env:
+            ANSIBLE_FORCE_COLOR: True
+            ANSIBLE_CALLBACKS_ENABLED: "profile_tasks"
+
+        - name: Move hpc variables back to group_vars
+          run: |
+            mv /tmp/hpc_*.yaml deploy/group_vars/
+
+        - name: Run ansible playbook (Full)
+          run: |
+            cd deploy
+            ansible-playbook ../playbook.yaml -i inventory.ini --connection=local -v
+          env:
+            ANSIBLE_FORCE_COLOR: True
+            ANSIBLE_CALLBACKS_ENABLED: "profile_tasks"

.github/workflows/lint.yaml

@@ -3,7 +3,17 @@ name: Ansible Lint
 
 on:
   push:
+    paths:
+      - 'roles/**'
+      - 'tasks/**'
+      - '.github/workflows/ansible-lint.yml'
+
   pull_request:
+    paths:
+      - 'roles/**'
+      - 'tasks/**'
+      - '.github/workflows/ansible-lint.yml'
+
 jobs:
   build:
     name: Ansible Lint

roles/apt_packages/tasks/main.yml

@@ -1,6 +1,7 @@
 ---
 - name: Ensure apt packages are installed
   become: true
+  timeout: 300
   ansible.builtin.apt:
     name: "{{ installed_packages }}"
     state: latest

roles/backups/tasks/backup.yaml

@@ -1,6 +1,7 @@
 ---
 - name: Ensure restic installed
   become: true
+  timeout: 300
   ansible.builtin.apt:
     name: restic
     state: latest

roles/cifs/handlers/main.yaml

@@ -5,6 +5,5 @@
     name: "{{ item }}"
     enabled: "yes"
     state: restarted
-    cmd: ""
   with_items:
     - smbd

roles/cifs/tasks/client.yaml

@@ -1,6 +1,7 @@
 ---
 - name: Install cifs
   become: true
+  timeout: 300
   ansible.builtin.apt:
     state: latest
     cache_valid_time: 3600

roles/cifs/tasks/server.yaml

@@ -1,6 +1,7 @@
 ---
 - name: Install samba
   become: true
+  timeout: 300
   ansible.builtin.apt:
     state: latest
     cache_valid_time: 3600
@@ -22,4 +23,4 @@
     owner: root
     group: root
     mode: "0644"
-  notify: restart services samba
+  notify: Restart services samba

roles/dask_gateway/handlers/main.yaml

@@ -5,6 +5,5 @@
     name: "{{ item }}"
     enabled: "yes"
     state: restarted
-    cmd: ""
   with_items:
     - dask-gateway

roles/dask_gateway/tasks/dask_gateway.yaml

@@ -49,7 +49,7 @@
     owner: dask
     group: dask
     mode: "0644"
-  notify: restart services dask-gateway
+  notify: Restart services dask-gateway
 
 - name: Copy the dask-gateway systemd service file
   become: true
@@ -77,7 +77,7 @@
     owner: root
     group: root
     mode: "0644"
-  notify: restart services dask-gateway
+  notify: Restart services dask-gateway
 
 - name: Ensure dask-gateway is enabled on boot
   become: true

roles/grafana/tasks/grafana.yaml

@@ -12,6 +12,7 @@
 
 - name: Install grafana
   become: true
+  timeout: 300
   ansible.builtin.apt:
     name: grafana{{ grafana_version }}
     state: "{% if grafana_version %}present{% else %}latest{% endif %}"

roles/jupyterhub/handlers/main.yaml

@@ -5,7 +5,6 @@
     name: "{{ item }}"
     enabled: "yes"
     state: restarted
-    cmd: ""
   with_items:
     - jupyterhub
 
@@ -15,7 +14,6 @@
     name: "{{ item }}"
     enabled: "yes"
     state: restarted
-    cmd: ""
   with_items:
     - jupyterhub-proxy
 
@@ -25,6 +23,5 @@
     name: "{{ item }}"
     enabled: "yes"
     state: restarted
-    cmd: ""
   with_items:
     - jupyterhub-ssh

roles/keycloak/tasks/keycloak.yaml

@@ -1,6 +1,7 @@
 ---
 - name: Install openjdk and python requirements
   become: true
+  timeout: 300
   ansible.builtin.apt:
     state: latest
     cache_valid_time: 3600
@@ -64,7 +65,8 @@
 
 - name: Ensure Keycloak admin user exists
   become: true
-  ansible.builtin.command: /opt/keycloak-{{ keycloak_version }}/bin/add-user-keycloak.sh -r master -u "{{ keycloak_admin_username }}" -p "{{ keycloak_admin_password
+  ansible.builtin.command:
+    /opt/keycloak-{{ keycloak_version }}/bin/add-user-keycloak.sh -r master -u "{{ keycloak_admin_username }}" -p "{{ keycloak_admin_password
     }}"
   args:
     creates: /opt/keycloak-{{ keycloak_version }}/standalone/configuration/keycloak-add-user.json

roles/mysql/defaults/main.yml

@@ -1,5 +1,7 @@
 ---
 mysql_enabled: false
+mysql_config_file: /etc/mysql/my.cnf
+
 mysql_databases:
   - slurm
   - conda-store
@@ -14,3 +16,10 @@ mysql_users:
   - username: conda-store
     password: eIbmUditL4RbQm0YPeLozRme
     privileges: "*.*:ALL"
+
+# Define a custom list of packages to install
+mysql_packages:
+  - mysql-server
+  - mysql-common
+
+mysql_python_package: python3-mysqldb

roles/mysql/handlers/main.yaml

@@ -5,6 +5,5 @@
     name: "{{ item }}"
     enabled: "yes"
     state: restarted
-    cmd: ""
   with_items:
     - mysql