diff --git a/.yamllint b/.yamllint index 8f1968740..ccf0eb89f 100644 --- a/.yamllint +++ b/.yamllint @@ -7,3 +7,5 @@ rules: level: warning truthy: allowed-values: ['true', 'false', 'yes', 'no'] + braces: + max-spaces-inside: 1 diff --git a/README.md b/README.md index 8e3c855ce..217133083 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,10 @@ master node ``` +If multiple hosts are in the master group, the playbook will automatically setup k3s in HA mode with etcd. +https://rancher.com/docs/k3s/latest/en/installation/ha-embedded/ +This requires at least k3s version 1.19.1 + If needed, you can also edit `inventory/my-cluster/group_vars/all.yml` to match your environment. Start provisioning of the cluster using the following command: diff --git a/inventory/sample/group_vars/all.yml b/inventory/sample/group_vars/all.yml index ada7200da..86c725a61 100644 --- a/inventory/sample/group_vars/all.yml +++ b/inventory/sample/group_vars/all.yml @@ -2,6 +2,18 @@ k3s_version: v1.22.3+k3s1 ansible_user: debian systemd_dir: /etc/systemd/system -master_ip: "{{ hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0]) }}" + +# If you define multiple masters you should be providing a loadbalanced +# apiserver endpoint to all masters here. This default value is only suitable +# for a non-HA setup, if used in a HA setup, it will not protect you if the +# first node fails. + +apiserver_endpoint: "{{ hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0]) }}" +apiserver_port: 6443 + +# Also you should define k3s_token so that masters can talk together securely + +# k3s_token: "mysupersecuretoken" + extra_server_args: "" extra_agent_args: "" diff --git a/roles/k3s/master/defaults/main.yml b/roles/k3s/master/defaults/main.yml index c56778f93..27960c94e 100644 --- a/roles/k3s/master/defaults/main.yml +++ b/roles/k3s/master/defaults/main.yml @@ -1,2 +1,13 @@ --- +ansible_user: root k3s_server_location: /var/lib/rancher/k3s +k3s_server_init_args: >- + {% if groups['master'] | length > 1 %} + {% if ansible_host == hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0]) %} + --cluster-init + {% else %} + --server https://{{ hostvars[groups['master'][0]]['ansible_host'] | default(groups['master'][0]) }}:6443 + {% endif %} + --token {{ k3s_token }} + {% endif %} + {{ extra_server_args | default('') }} diff --git a/roles/k3s/master/tasks/main.yml b/roles/k3s/master/tasks/main.yml index c19686335..219106077 100644 --- a/roles/k3s/master/tasks/main.yml +++ b/roles/k3s/master/tasks/main.yml @@ -1,4 +1,39 @@ --- +- name: Clean previous runs of k3s-init + ansible.builtin.systemd: + name: k3s-init + state: stopped + failed_when: false + +- name: Clean previous runs of k3s-init + ansible.builtin.command: systemctl reset-failed k3s-init # noqa: command-instead-of-module + failed_when: false + changed_when: false + +- name: Init cluster inside the transient k3s-init service + ansible.builtin.command: + cmd: "systemd-run -p RestartSec=2 \ + -p Restart=on-failure \ + --unit=k3s-init \ + k3s server {{ k3s_server_init_args }}" + creates: "{{ systemd_dir }}/k3s.service" + +- name: Verification + block: + - name: Verify that all nodes actually joined (check k3s-init.service if this fails) + ansible.builtin.command: + cmd: k3s kubectl get nodes -l "node-role.kubernetes.io/master=true" -o=jsonpath="{.items[*].metadata.name}" + register: nodes + until: nodes.rc == 0 and (nodes.stdout.split() | length) == (groups['master'] | length) + retries: 20 + delay: 10 + changed_when: false + always: + - name: Kill the temporary service used for initialization + ansible.builtin.systemd: + name: k3s-init + state: stopped + failed_when: false - name: Copy K3s service file register: k3s_service @@ -22,17 +57,17 @@ - name: Register node-token file access mode ansible.builtin.stat: - path: "{{ k3s_server_location }}/server/node-token" + path: "{{ k3s_server_location }}/server" register: p - name: Change file access node-token ansible.builtin.file: - path: "{{ k3s_server_location }}/server/node-token" + path: "{{ k3s_server_location }}/server" mode: "g+rx,o+rx" - name: Read node-token from master ansible.builtin.slurp: - path: "{{ k3s_server_location }}/server/node-token" + src: "{{ k3s_server_location }}/server/node-token" register: node_token - name: Store Master node-token @@ -41,7 +76,7 @@ - name: Restore node-token file access ansible.builtin.file: - path: "{{ k3s_server_location }}/server/node-token" + path: "{{ k3s_server_location }}/server" mode: "{{ p.stat.mode }}" - name: Create directory .kube @@ -59,10 +94,10 @@ owner: "{{ ansible_user }}" mode: "u=rw,g=,o=" -- name: Replace https://localhost:6443 by https://master-ip:6443 +- name: Configure kubectl cluster to server endpoint ansible.builtin.command: >- /usr/local/bin/k3s kubectl config set-cluster default - --server=https://{{ master_ip }}:6443 + --server=https://{{ apiserver_endpoint }}:{{ apiserver_port | default(6443) }} --kubeconfig ~{{ ansible_user }}/.kube/config changed_when: true diff --git a/roles/k3s/node/templates/k3s.service.j2 b/roles/k3s/node/templates/k3s.service.j2 index 99a0ac3d0..37d6e9672 100644 --- a/roles/k3s/node/templates/k3s.service.j2 +++ b/roles/k3s/node/templates/k3s.service.j2 @@ -7,7 +7,7 @@ After=network-online.target Type=notify ExecStartPre=-/sbin/modprobe br_netfilter ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/local/bin/k3s agent --server https://{{ master_ip }}:6443 --token {{ hostvars[groups['master'][0]]['token'] }} {{ extra_agent_args | default("") }} +ExecStart=/usr/local/bin/k3s agent --server https://{{ apiserver_endpoint }}:{{ apiserver_port | default(6443) }} --token {{ hostvars[groups['master'][0]]['token'] | default(k3s_token) }} {{ extra_agent_args | default("") }} KillMode=process Delegate=yes # Having non-zero Limit*s causes performance problems due to accounting overhead diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml index 0ec2c5caf..a875c9f10 100644 --- a/roles/reset/tasks/main.yml +++ b/roles/reset/tasks/main.yml @@ -8,6 +8,7 @@ with_items: - k3s - k3s-node + - k3s-init - name: Pkill k3s container runtimes" register: pkill_containerd_shim_runc