Support HA mode with embedded DB

.yamllint

@@ -7,3 +7,5 @@ rules:
     level: warning
   truthy:
     allowed-values: ['true', 'false', 'yes', 'no']
+  braces:
+    max-spaces-inside: 1

README.md

@@ -43,6 +43,10 @@ master
 node
 ```
 
+If multiple hosts are in the master group, the playbook will automatically setup k3s in HA mode with etcd.
+https://rancher.com/docs/k3s/latest/en/installation/ha-embedded/
+This requires at least k3s version 1.19.1
+
 If needed, you can also edit `inventory/my-cluster/group_vars/all.yml` to match your environment.
 
 Start provisioning of the cluster using the following command:

inventory/sample/group_vars/all.yml

@@ -2,6 +2,18 @@
 k3s_version: v1.22.3+k3s1
 ansible_user: debian
 systemd_dir: /etc/systemd/system
-master_ip: "{{ hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0]) }}"
+
+# If you define multiple masters you should be providing a loadbalanced
+# apiserver endpoint to all masters here. This default value is only suitable
+# for a non-HA setup, if used in a HA setup, it will not protect you if the
+# first node fails.
+
+apiserver_endpoint: "{{ hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0]) }}"
+apiserver_port: 6443
+
+# Also you should define k3s_token so that masters can talk together securely
+
+# k3s_token: "mysupersecuretoken"
+
 extra_server_args: ""
 extra_agent_args: ""

roles/k3s/master/defaults/main.yml

@@ -1,2 +1,13 @@
 ---
+ansible_user: root
 k3s_server_location: /var/lib/rancher/k3s
+k3s_server_init_args: >-
+  {% if groups['master'] | length > 1 %}
+    {% if ansible_host == hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0]) %}
+      --cluster-init
+    {% else %}
+      --server https://{{ hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0]) }}:6443
+    {% endif %}
+    --token {{ k3s_token }}
+  {% endif %}
+  {{ extra_server_args | default('') }}

roles/k3s/master/tasks/main.yml

@@ -1,4 +1,39 @@
 ---
+- name: Clean previous runs of k3s-init
+  ansible.builtin.systemd:
+    name: k3s-init
+    state: stopped
+  failed_when: false
+
+- name: Clean previous runs of k3s-init
+  ansible.builtin.command: systemctl reset-failed k3s-init  # noqa: command-instead-of-module
+  failed_when: false
+  changed_when: false
+
+- name: Init cluster inside the transient k3s-init service
+  ansible.builtin.command:
+    cmd: "systemd-run -p RestartSec=2 \
+                      -p Restart=on-failure \
+                      --unit=k3s-init \
+                      k3s server {{ k3s_server_init_args }}"
+    creates: "{{ systemd_dir }}/k3s.service"
+
+- name: Verification
+  block:
+    - name: Verify that all nodes actually joined (check k3s-init.service if this fails)
+      ansible.builtin.command:
+        cmd: k3s kubectl get nodes -l "node-role.kubernetes.io/master=true" -o=jsonpath="{.items[*].metadata.name}"
+      register: nodes
+      until: nodes.rc == 0 and (nodes.stdout.split() | length) == (groups['master'] | length)
+      retries: 20
+      delay: 10
+      changed_when: false
+  always:
+    - name: Kill the temporary service used for initialization
+      ansible.builtin.systemd:
+        name: k3s-init
+        state: stopped
+      failed_when: false
 
 - name: Copy K3s service file
   register: k3s_service
@@ -22,17 +57,17 @@
 
 - name: Register node-token file access mode
   ansible.builtin.stat:
-    path: "{{ k3s_server_location }}/server/node-token"
+    path: "{{ k3s_server_location }}/server"
   register: p
 
 - name: Change file access node-token
   ansible.builtin.file:
-    path: "{{ k3s_server_location }}/server/node-token"
+    path: "{{ k3s_server_location }}/server"
     mode: "g+rx,o+rx"
 
 - name: Read node-token from master
   ansible.builtin.slurp:
-    path: "{{ k3s_server_location }}/server/node-token"
+    src: "{{ k3s_server_location }}/server/node-token"
   register: node_token
 
 - name: Store Master node-token
@@ -41,7 +76,7 @@
 
 - name: Restore node-token file access
   ansible.builtin.file:
-    path: "{{ k3s_server_location }}/server/node-token"
+    path: "{{ k3s_server_location }}/server"
     mode: "{{ p.stat.mode }}"
 
 - name: Create directory .kube
@@ -59,10 +94,10 @@
     owner: "{{ ansible_user }}"
     mode: "u=rw,g=,o="
 
-- name: Replace https://localhost:6443 by https://master-ip:6443
+- name: Configure kubectl cluster to server endpoint
   ansible.builtin.command: >-
     /usr/local/bin/k3s kubectl config set-cluster default
-      --server=https://{{ master_ip }}:6443
+      --server=https://{{ apiserver_endpoint }}:{{ apiserver_port | default(6443) }}
       --kubeconfig ~{{ ansible_user }}/.kube/config
   changed_when: true
 

roles/k3s/node/templates/k3s.service.j2

@@ -7,7 +7,7 @@ After=network-online.target
 Type=notify
 ExecStartPre=-/sbin/modprobe br_netfilter
 ExecStartPre=-/sbin/modprobe overlay
-ExecStart=/usr/local/bin/k3s agent --server https://{{ master_ip }}:6443 --token {{ hostvars[groups['master'][0]]['token'] }} {{ extra_agent_args | default("") }}
+ExecStart=/usr/local/bin/k3s agent --server https://{{ apiserver_endpoint }}:{{ apiserver_port | default(6443) }} --token {{ hostvars[groups['master'][0]]['token'] | default(k3s_token) }} {{ extra_agent_args | default("") }}
 KillMode=process
 Delegate=yes
 # Having non-zero Limit*s causes performance problems due to accounting overhead

roles/reset/tasks/main.yml

@@ -8,6 +8,7 @@
   with_items:
     - k3s
     - k3s-node
+    - k3s-init
 
 - name: Pkill k3s container runtimes"
   register: pkill_containerd_shim_runc