Making it functional + adding libvirt configuration

.gitignore

@@ -1,6 +1,7 @@
 # General
 .vagrant/
 configs/
+settings.yaml
 
 # Log files (if you are creating logs in debug mode, uncomment this)
 # *.log
@@ -30,4 +31,6 @@ Icon
 .AppleDesktop
 Network Trash Folder
 Temporary Items
-.apdisk
+.apdisk
+
+scripts/customize-*.sh

README.md

@@ -29,7 +29,8 @@ The following are the best bundles to **save up to $419** with code **EARTHDAY24
 
 ## Setup Prerequisites
 
-- A working Vagrant setup using Vagrant + VirtualBox
+- A working Vagrant setup using Vagrant with VirtualBox or libvirt available
+- If using libvirt user needs to be added to libvirt group (`sudo usermod -aG libvirt $(whoami)`)
 
 ## Documentation
 
@@ -45,7 +46,35 @@ Refer to this link for documentation full: https://devopscube.com/kubernetes-clu
 1. Working Vagrant setup
 2. 8 Gig + RAM workstation as the Vms use 3 vCPUS and 4+ GB RAM
 
-## For MAC/Linux Users
+
+## Firewall Configuration
+
+To allow virtual machines to access NFS server on the host machine you will probably need to update firewall rules.
+
+### Rules for VirtualBox
+
+```shell
+sudo firewall-cmd --permanent --new-zone=virtualbox \
+  && sudo firewall-cmd --permanent --zone=virtualbox --add-interface=vboxnet0 \
+  && sudo firewall-cmd --permanent --zone=virtualbox --add-interface=vboxnet1 \
+  && sudo firewall-cmd --permanent --zone=virtualbox --add-service=nfs3 \
+  && sudo firewall-cmd --permanent --zone=virtualbox --add-service=nfs \
+  && sudo firewall-cmd --permanent --zone=virtualbox --add-service=rpc-bind \
+  && sudo firewall-cmd --permanent --zone=virtualbox --add-service=mountd \
+  && sudo firewall-cmd --reload
+```
+
+### Rules for libvirt
+
+```shell
+sudo firewall-cmd --permanent --zone=libvirt --add-service=nfs3 \
+  && sudo firewall-cmd --permanent --zone=libvirt --add-service=nfs \
+  && sudo firewall-cmd --permanent --zone=libvirt --add-service=rpc-bind \
+  && sudo firewall-cmd --permanent --zone=libvirt --add-service=mountd \
+  && sudo firewall-cmd --reload
+```
+
+## For MAC/Linux Users with VirtualBox
 
 The latest version of Virtualbox for Mac/Linux can cause issues.
 
@@ -69,20 +98,33 @@ To provision the cluster, execute the following commands.
 ```shell
 git clone https://github.com/scriptcamp/vagrant-kubeadm-kubernetes.git
 cd vagrant-kubeadm-kubernetes
-vagrant up
+cp settings.yaml.sample settings.yaml
+```
+
+### With VirtualBox
+
+```shell
+vagrant up --provider virtualbox
 ```
+
+### With libvirt
+
+```shell
+vagrant up --provider libvirt
+```
+
 ## Set Kubeconfig file variable
 
 ```shell
 cd vagrant-kubeadm-kubernetes
-cd configs
-export KUBECONFIG=$(pwd)/config
+
+export KUBECONFIG=$(pwd)/configs/config
 ```
 
 or you can copy the config file to .kube directory.
 
 ```shell
-cp config ~/.kube/
+cp configs/config ~/.kube/
 ```
 
 ## Install Kubernetes Dashboard

Vagrantfile

@@ -2,6 +2,7 @@
 require "yaml"
 vagrant_root = File.dirname(File.expand_path(__FILE__))
 settings = YAML.load_file "#{vagrant_root}/settings.yaml"
+autostart = settings["autostart"]
 
 IP_SECTIONS = settings["network"]["control_ip"].match(/^([0-9.]+\.)([^.]+)$/)
 # First 3 octets including the trailing dot:
@@ -11,6 +12,15 @@ IP_START = Integer(IP_SECTIONS.captures[1])
 NUM_WORKER_NODES = settings["nodes"]["workers"]["count"]
 
 Vagrant.configure("2") do |config|
+  config.vm.synced_folder "./", "/vagrant", type: "nfs", nfs_version: 4
+
+  config.vm.provider "libvirt" do |libvirt|
+      libvirt.uri = "qemu:///system"
+      libvirt.driver = "qemu"
+      libvirt.management_network_autostart = autostart
+      libvirt.autostart = autostart
+  end
+
   config.vm.provision "shell", env: { "IP_NW" => IP_NW, "IP_START" => IP_START, "NUM_WORKER_NODES" => NUM_WORKER_NODES }, inline: <<-SHELL
       apt-get update -y
       echo "$IP_NW$((IP_START)) controlplane" >> /etc/hosts
@@ -28,28 +38,46 @@ Vagrant.configure("2") do |config|
 
   config.vm.define "controlplane" do |controlplane|
     controlplane.vm.hostname = "controlplane"
-    controlplane.vm.network "private_network", ip: settings["network"]["control_ip"]
+    controlplane.vm.network "private_network", ip: settings["network"]["control_ip"], autostart: autostart
+
     if settings["shared_folders"]
       settings["shared_folders"].each do |shared_folder|
         controlplane.vm.synced_folder shared_folder["host_path"], shared_folder["vm_path"]
       end
     end
+
     controlplane.vm.provider "virtualbox" do |vb|
         vb.cpus = settings["nodes"]["control"]["cpu"]
         vb.memory = settings["nodes"]["control"]["memory"]
+
         if settings["cluster_name"] and settings["cluster_name"] != ""
           vb.customize ["modifyvm", :id, "--groups", ("/" + settings["cluster_name"])]
         end
+
+        if autostart
+          vb.customize ["modifyvm", :id, "--autostart-enabled", "on"]
+          vb.customize ["modifyvm", :id, "--autostop-type", "acpishutdown"]
+        end
     end
+
+    controlplane.vm.provider "libvirt" do |vb|
+        vb.cpus = settings["nodes"]["control"]["cpu"]
+        vb.memory = settings["nodes"]["control"]["memory"]
+    end
+
     controlplane.vm.provision "shell",
       env: {
         "DNS_SERVERS" => settings["network"]["dns_servers"].join(" "),
         "ENVIRONMENT" => settings["environment"],
         "KUBERNETES_VERSION" => settings["software"]["kubernetes"],
         "KUBERNETES_VERSION_SHORT" => settings["software"]["kubernetes"][0..3],
-        "OS" => settings["software"]["os"]
+        "OS" => settings["software"]["os"],
+        "CRI" => settings["software"]["cri"],
+        "CRI_VERSION" => settings["software"]["cri-version"],
+        "CRI_VERSION_SHORT" => settings["software"]["cri-version"][0..3],
       },
       path: "scripts/common.sh"
+
     controlplane.vm.provision "shell",
       env: {
         "CALICO_VERSION" => settings["software"]["calico"],
@@ -64,30 +92,52 @@ Vagrant.configure("2") do |config|
 
     config.vm.define "node0#{i}" do |node|
       node.vm.hostname = "node0#{i}"
-      node.vm.network "private_network", ip: IP_NW + "#{IP_START + i}"
+      node.vm.network "private_network", ip: IP_NW + "#{IP_START + i}", autostart: autostart
+
       if settings["shared_folders"]
         settings["shared_folders"].each do |shared_folder|
           node.vm.synced_folder shared_folder["host_path"], shared_folder["vm_path"]
         end
       end
+
       node.vm.provider "virtualbox" do |vb|
           vb.cpus = settings["nodes"]["workers"]["cpu"]
           vb.memory = settings["nodes"]["workers"]["memory"]
+
           if settings["cluster_name"] and settings["cluster_name"] != ""
             vb.customize ["modifyvm", :id, "--groups", ("/" + settings["cluster_name"])]
           end
+
+          if autostart
+            vb.customize ["modifyvm", :id, "--autostart-enabled", "on"]
+            vb.customize ["modifyvm", :id, "--autostop-type", "acpishutdown"]
+          end
       end
+
+      node.vm.provider "libvirt" do |vb|
+          vb.cpus = settings["nodes"]["workers"]["cpu"]
+          vb.memory = settings["nodes"]["workers"]["memory"]
+      end
+
       node.vm.provision "shell",
         env: {
           "DNS_SERVERS" => settings["network"]["dns_servers"].join(" "),
           "ENVIRONMENT" => settings["environment"],
           "KUBERNETES_VERSION" => settings["software"]["kubernetes"],
           "KUBERNETES_VERSION_SHORT" => settings["software"]["kubernetes"][0..3],
-          "OS" => settings["software"]["os"]
+          "OS" => settings["software"]["os"],
+          "CRI" => settings["software"]["cri"],
+          "CRI_VERSION" => settings["software"]["cri-version"],
+          "CRI_VERSION_SHORT" => settings["software"]["cri-version"][0..3],
         },
         path: "scripts/common.sh"
+
       node.vm.provision "shell", path: "scripts/node.sh"
 
+      if File.exist?("scripts/customize-node.sh")
+        node.vm.provision "shell", path: "scripts/customize-node.sh"
+      end
+
       # Only install the dashboard after provisioning the last worker (and when enabled).
       if i == NUM_WORKER_NODES and settings["software"]["dashboard"] and settings["software"]["dashboard"] != ""
         node.vm.provision "shell", path: "scripts/dashboard.sh"

scripts/common.sh

@@ -4,81 +4,118 @@
 
 set -euxo pipefail
 
+# Check if /vagrant is correctly mounted through NFS
+if ! mount | grep -q 'on /vagrant type nfs'; then
+  echo "/vagrant is not mounted through NFS, check your firewall settings"
+  exit 1
+fi
+
 # Variable Declaration
 
 # DNS Setting
 if [ ! -d /etc/systemd/resolved.conf.d ]; then
-	sudo mkdir /etc/systemd/resolved.conf.d/
+	mkdir /etc/systemd/resolved.conf.d/
 fi
-cat <<EOF | sudo tee /etc/systemd/resolved.conf.d/dns_servers.conf
+cat <<EOF | tee /etc/systemd/resolved.conf.d/dns_servers.conf
 [Resolve]
 DNS=${DNS_SERVERS}
 EOF
 
-sudo systemctl restart systemd-resolved
+systemctl restart systemd-resolved
 
 # disable swap
-sudo swapoff -a
+swapoff -a
 
-# keeps the swaf off during reboot
+# keeps the swap off during reboot
 (crontab -l 2>/dev/null; echo "@reboot /sbin/swapoff -a") | crontab - || true
-sudo apt-get update -y
+apt-get update -y
 
 
 # Create the .conf file to load the modules at bootup
-cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
+cat <<EOF | tee /etc/modules-load.d/k8s.conf
 overlay
 br_netfilter
 EOF
 
-sudo modprobe overlay
-sudo modprobe br_netfilter
+modprobe overlay
+modprobe br_netfilter
 
 # sysctl params required by setup, params persist across reboots
-cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
+cat <<EOF | tee /etc/sysctl.d/k8s.conf
 net.bridge.bridge-nf-call-iptables  = 1
 net.bridge.bridge-nf-call-ip6tables = 1
 net.ipv4.ip_forward                 = 1
 EOF
 
 # Apply sysctl params without reboot
-sudo sysctl --system
+sysctl --system
 
-## Install CRIO Runtime
-
-sudo apt-get update -y
+apt-get update -y
 apt-get install -y software-properties-common curl apt-transport-https ca-certificates
 
-curl -fsSL https://pkgs.k8s.io/addons:/cri-o:/prerelease:/main/deb/Release.key |
-    gpg --dearmor -o /etc/apt/keyrings/cri-o-apt-keyring.gpg
-echo "deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/prerelease:/main/deb/ /" |
-    tee /etc/apt/sources.list.d/cri-o.list
+if [ "$CRI" = "cri-o" ]; then
+  ## Install CRIO Runtime
+
+  curl -fsSL https://pkgs.k8s.io/addons:/cri-o:/stable:/v$CRI_VERSION_SHORT/deb/Release.key |
+      gpg --dearmor -o /etc/apt/keyrings/cri-o-apt-keyring.gpg
+
+  echo "deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/stable:/v$CRI_VERSION_SHORT/deb/ /" |
+      tee /etc/apt/sources.list.d/cri-o.list
+
+  apt-get update -y
+  apt-get install -y cri-o="$CRI_VERSION"
+
+  systemctl daemon-reload
+  systemctl enable crio --now
+  systemctl start crio.service
+
+  apt-mark hold cri-o
 
-sudo apt-get update -y
-sudo apt-get install -y cri-o
+  echo "CRI-O runtime installed successfully"
+fi
+
+if [ "$CRI" = "containerd" ]; then
+  ## Install containerd Runtime
+
+  curl -fsSL https://download.docker.com/linux/ubuntu/gpg |
+      gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+
+  echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" |
+      tee /etc/apt/sources.list.d/docker.list
 
-sudo systemctl daemon-reload
-sudo systemctl enable crio --now
-sudo systemctl start crio.service
+  # Update and install containerd
+  apt-get update -y
+  apt-get install -y containerd.io
 
-echo "CRI runtime installed successfully"
+  mkdir -p /etc/containerd
+  containerd config default > /etc/containerd/config.toml
+
+  systemctl daemon-reload
+  systemctl enable containerd --now
+  systemctl start containerd
+
+  apt-mark hold containerd.io
+
+  echo "containerd runtime installed successfully"
+fi
 
-sudo mkdir -p /etc/apt/keyrings
-curl -fsSL https://pkgs.k8s.io/core:/stable:/v$KUBERNETES_VERSION_SHORT/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
-echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v$KUBERNETES_VERSION_SHORT/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list
+mkdir -p /etc/apt/keyrings
+curl -fsSL https://pkgs.k8s.io/core:/stable:/v$KUBERNETES_VERSION_SHORT/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v$KUBERNETES_VERSION_SHORT/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
 
 
-sudo apt-get update -y
-sudo apt-get install -y kubelet="$KUBERNETES_VERSION" kubectl="$KUBERNETES_VERSION" kubeadm="$KUBERNETES_VERSION"
-sudo apt-get update -y
-sudo apt-get install -y jq
+apt-get update -y
+apt-get install -y kubelet="$KUBERNETES_VERSION" kubectl="$KUBERNETES_VERSION" kubeadm="$KUBERNETES_VERSION"
+apt-get update -y
+apt-get install -y jq
 
 # Disable auto-update services
-sudo apt-mark hold kubelet kubectl kubeadm cri-o
+apt-mark hold kubelet kubectl kubeadm
 
+local_ip="$(ip --json a s | jq -r '.[] | select(.ifname == "eth1") | .addr_info[] | select(.family == "inet") | .local')"
 
-local_ip="$(ip --json a s | jq -r '.[] | if .ifname == "eth1" then .addr_info[] | if .family == "inet" then .local else empty end else empty end')"
+mkdir -p /etc/kubernetes/kubelet.conf.d
 cat > /etc/default/kubelet << EOF
-KUBELET_EXTRA_ARGS=--node-ip=$local_ip
+KUBELET_EXTRA_ARGS="--node-ip=$local_ip --config-dir=/etc/kubernetes/kubelet.conf.d"
 ${ENVIRONMENT}
 EOF