diff --git a/.gitignore b/.gitignore index 582c81c..27eebf4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # General .vagrant/ configs/ +settings.yaml # Log files (if you are creating logs in debug mode, uncomment this) # *.log @@ -30,4 +31,6 @@ Icon .AppleDesktop Network Trash Folder Temporary Items -.apdisk \ No newline at end of file +.apdisk + +scripts/customize-*.sh \ No newline at end of file diff --git a/README.md b/README.md index f47ce86..b8778f2 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,8 @@ The following are the best bundles to **save up to $419** with code **EARTHDAY24 ## Setup Prerequisites -- A working Vagrant setup using Vagrant + VirtualBox +- A working Vagrant setup using Vagrant with VirtualBox or libvirt available +- If using libvirt user needs to be added to libvirt group (`sudo usermod -aG libvirt $(whoami)`) ## Documentation @@ -45,7 +46,35 @@ Refer to this link for documentation full: https://devopscube.com/kubernetes-clu 1. Working Vagrant setup 2. 8 Gig + RAM workstation as the Vms use 3 vCPUS and 4+ GB RAM -## For MAC/Linux Users + +## Firewall Configuration + +To allow virtual machines to access NFS server on the host machine you will probably need to update firewall rules. + +### Rules for VirtualBox + +```shell +sudo firewall-cmd --permanent --new-zone=virtualbox \ + && sudo firewall-cmd --permanent --zone=virtualbox --add-interface=vboxnet0 \ + && sudo firewall-cmd --permanent --zone=virtualbox --add-interface=vboxnet1 \ + && sudo firewall-cmd --permanent --zone=virtualbox --add-service=nfs3 \ + && sudo firewall-cmd --permanent --zone=virtualbox --add-service=nfs \ + && sudo firewall-cmd --permanent --zone=virtualbox --add-service=rpc-bind \ + && sudo firewall-cmd --permanent --zone=virtualbox --add-service=mountd \ + && sudo firewall-cmd --reload +``` + +### Rules for libvirt + +```shell +sudo firewall-cmd --permanent --zone=libvirt --add-service=nfs3 \ + && sudo firewall-cmd --permanent --zone=libvirt --add-service=nfs \ + && sudo firewall-cmd --permanent --zone=libvirt --add-service=rpc-bind \ + && sudo firewall-cmd --permanent --zone=libvirt --add-service=mountd \ + && sudo firewall-cmd --reload +``` + +## For MAC/Linux Users with VirtualBox The latest version of Virtualbox for Mac/Linux can cause issues. @@ -69,20 +98,33 @@ To provision the cluster, execute the following commands. ```shell git clone https://github.com/scriptcamp/vagrant-kubeadm-kubernetes.git cd vagrant-kubeadm-kubernetes -vagrant up +cp settings.yaml.sample settings.yaml +``` + +### With VirtualBox + +```shell +vagrant up --provider virtualbox ``` + +### With libvirt + +```shell +vagrant up --provider libvirt +``` + ## Set Kubeconfig file variable ```shell cd vagrant-kubeadm-kubernetes -cd configs -export KUBECONFIG=$(pwd)/config + +export KUBECONFIG=$(pwd)/configs/config ``` or you can copy the config file to .kube directory. ```shell -cp config ~/.kube/ +cp configs/config ~/.kube/ ``` ## Install Kubernetes Dashboard diff --git a/Vagrantfile b/Vagrantfile index 2803482..2ecf835 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -2,6 +2,7 @@ require "yaml" vagrant_root = File.dirname(File.expand_path(__FILE__)) settings = YAML.load_file "#{vagrant_root}/settings.yaml" +autostart = settings["autostart"] IP_SECTIONS = settings["network"]["control_ip"].match(/^([0-9.]+\.)([^.]+)$/) # First 3 octets including the trailing dot: @@ -11,6 +12,15 @@ IP_START = Integer(IP_SECTIONS.captures[1]) NUM_WORKER_NODES = settings["nodes"]["workers"]["count"] Vagrant.configure("2") do |config| + config.vm.synced_folder "./", "/vagrant", type: "nfs", nfs_version: 4 + + config.vm.provider "libvirt" do |libvirt| + libvirt.uri = "qemu:///system" + libvirt.driver = "qemu" + libvirt.management_network_autostart = autostart + libvirt.autostart = autostart + end + config.vm.provision "shell", env: { "IP_NW" => IP_NW, "IP_START" => IP_START, "NUM_WORKER_NODES" => NUM_WORKER_NODES }, inline: <<-SHELL apt-get update -y echo "$IP_NW$((IP_START)) controlplane" >> /etc/hosts @@ -28,28 +38,46 @@ Vagrant.configure("2") do |config| config.vm.define "controlplane" do |controlplane| controlplane.vm.hostname = "controlplane" - controlplane.vm.network "private_network", ip: settings["network"]["control_ip"] + controlplane.vm.network "private_network", ip: settings["network"]["control_ip"], autostart: autostart + if settings["shared_folders"] settings["shared_folders"].each do |shared_folder| controlplane.vm.synced_folder shared_folder["host_path"], shared_folder["vm_path"] end end + controlplane.vm.provider "virtualbox" do |vb| vb.cpus = settings["nodes"]["control"]["cpu"] vb.memory = settings["nodes"]["control"]["memory"] + if settings["cluster_name"] and settings["cluster_name"] != "" vb.customize ["modifyvm", :id, "--groups", ("/" + settings["cluster_name"])] end + + if autostart + vb.customize ["modifyvm", :id, "--autostart-enabled", "on"] + vb.customize ["modifyvm", :id, "--autostop-type", "acpishutdown"] + end end + + controlplane.vm.provider "libvirt" do |vb| + vb.cpus = settings["nodes"]["control"]["cpu"] + vb.memory = settings["nodes"]["control"]["memory"] + end + controlplane.vm.provision "shell", env: { "DNS_SERVERS" => settings["network"]["dns_servers"].join(" "), "ENVIRONMENT" => settings["environment"], "KUBERNETES_VERSION" => settings["software"]["kubernetes"], "KUBERNETES_VERSION_SHORT" => settings["software"]["kubernetes"][0..3], - "OS" => settings["software"]["os"] + "OS" => settings["software"]["os"], + "CRI" => settings["software"]["cri"], + "CRI_VERSION" => settings["software"]["cri-version"], + "CRI_VERSION_SHORT" => settings["software"]["cri-version"][0..3], }, path: "scripts/common.sh" + controlplane.vm.provision "shell", env: { "CALICO_VERSION" => settings["software"]["calico"], @@ -64,30 +92,52 @@ Vagrant.configure("2") do |config| config.vm.define "node0#{i}" do |node| node.vm.hostname = "node0#{i}" - node.vm.network "private_network", ip: IP_NW + "#{IP_START + i}" + node.vm.network "private_network", ip: IP_NW + "#{IP_START + i}", autostart: autostart + if settings["shared_folders"] settings["shared_folders"].each do |shared_folder| node.vm.synced_folder shared_folder["host_path"], shared_folder["vm_path"] end end + node.vm.provider "virtualbox" do |vb| vb.cpus = settings["nodes"]["workers"]["cpu"] vb.memory = settings["nodes"]["workers"]["memory"] + if settings["cluster_name"] and settings["cluster_name"] != "" vb.customize ["modifyvm", :id, "--groups", ("/" + settings["cluster_name"])] end + + if autostart + vb.customize ["modifyvm", :id, "--autostart-enabled", "on"] + vb.customize ["modifyvm", :id, "--autostop-type", "acpishutdown"] + end end + + node.vm.provider "libvirt" do |vb| + vb.cpus = settings["nodes"]["workers"]["cpu"] + vb.memory = settings["nodes"]["workers"]["memory"] + end + node.vm.provision "shell", env: { "DNS_SERVERS" => settings["network"]["dns_servers"].join(" "), "ENVIRONMENT" => settings["environment"], "KUBERNETES_VERSION" => settings["software"]["kubernetes"], "KUBERNETES_VERSION_SHORT" => settings["software"]["kubernetes"][0..3], - "OS" => settings["software"]["os"] + "OS" => settings["software"]["os"], + "CRI" => settings["software"]["cri"], + "CRI_VERSION" => settings["software"]["cri-version"], + "CRI_VERSION_SHORT" => settings["software"]["cri-version"][0..3], }, path: "scripts/common.sh" + node.vm.provision "shell", path: "scripts/node.sh" + if File.exist?("scripts/customize-node.sh") + node.vm.provision "shell", path: "scripts/customize-node.sh" + end + # Only install the dashboard after provisioning the last worker (and when enabled). if i == NUM_WORKER_NODES and settings["software"]["dashboard"] and settings["software"]["dashboard"] != "" node.vm.provision "shell", path: "scripts/dashboard.sh" diff --git a/scripts/common.sh b/scripts/common.sh index c261cc7..cd5b516 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -4,81 +4,118 @@ set -euxo pipefail +# Check if /vagrant is correctly mounted through NFS +if ! mount | grep -q 'on /vagrant type nfs'; then + echo "/vagrant is not mounted through NFS, check your firewall settings" + exit 1 +fi + # Variable Declaration # DNS Setting if [ ! -d /etc/systemd/resolved.conf.d ]; then - sudo mkdir /etc/systemd/resolved.conf.d/ + mkdir /etc/systemd/resolved.conf.d/ fi -cat </dev/null; echo "@reboot /sbin/swapoff -a") | crontab - || true -sudo apt-get update -y +apt-get update -y # Create the .conf file to load the modules at bootup -cat < /etc/containerd/config.toml + + systemctl daemon-reload + systemctl enable containerd --now + systemctl start containerd + + apt-mark hold containerd.io + + echo "containerd runtime installed successfully" +fi -sudo mkdir -p /etc/apt/keyrings -curl -fsSL https://pkgs.k8s.io/core:/stable:/v$KUBERNETES_VERSION_SHORT/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg -echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v$KUBERNETES_VERSION_SHORT/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list +mkdir -p /etc/apt/keyrings +curl -fsSL https://pkgs.k8s.io/core:/stable:/v$KUBERNETES_VERSION_SHORT/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg +echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v$KUBERNETES_VERSION_SHORT/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list -sudo apt-get update -y -sudo apt-get install -y kubelet="$KUBERNETES_VERSION" kubectl="$KUBERNETES_VERSION" kubeadm="$KUBERNETES_VERSION" -sudo apt-get update -y -sudo apt-get install -y jq +apt-get update -y +apt-get install -y kubelet="$KUBERNETES_VERSION" kubectl="$KUBERNETES_VERSION" kubeadm="$KUBERNETES_VERSION" +apt-get update -y +apt-get install -y jq # Disable auto-update services -sudo apt-mark hold kubelet kubectl kubeadm cri-o +apt-mark hold kubelet kubectl kubeadm +local_ip="$(ip --json a s | jq -r '.[] | select(.ifname == "eth1") | .addr_info[] | select(.family == "inet") | .local')" -local_ip="$(ip --json a s | jq -r '.[] | if .ifname == "eth1" then .addr_info[] | if .family == "inet" then .local else empty end else empty end')" +mkdir -p /etc/kubernetes/kubelet.conf.d cat > /etc/default/kubelet << EOF -KUBELET_EXTRA_ARGS=--node-ip=$local_ip +KUBELET_EXTRA_ARGS="--node-ip=$local_ip --config-dir=/etc/kubernetes/kubelet.conf.d" ${ENVIRONMENT} EOF diff --git a/scripts/master.sh b/scripts/master.sh index 4582c92..cddbf5b 100644 --- a/scripts/master.sh +++ b/scripts/master.sh @@ -6,15 +6,15 @@ set -euxo pipefail NODENAME=$(hostname -s) -sudo kubeadm config images pull +kubeadm config images pull echo "Preflight Check Passed: Downloaded All Required Images" -sudo kubeadm init --apiserver-advertise-address=$CONTROL_IP --apiserver-cert-extra-sans=$CONTROL_IP --pod-network-cidr=$POD_CIDR --service-cidr=$SERVICE_CIDR --node-name "$NODENAME" --ignore-preflight-errors Swap +kubeadm init --apiserver-advertise-address=$CONTROL_IP --apiserver-cert-extra-sans=$CONTROL_IP --pod-network-cidr=$POD_CIDR --service-cidr=$SERVICE_CIDR --node-name "$NODENAME" --ignore-preflight-errors Swap mkdir -p "$HOME"/.kube -sudo cp -i /etc/kubernetes/admin.conf "$HOME"/.kube/config -sudo chown "$(id -u)":"$(id -g)" "$HOME"/.kube/config +cp -i /etc/kubernetes/admin.conf "$HOME"/.kube/config +chown "$(id -u)":"$(id -g)" "$HOME"/.kube/config # Save Configs to shared /Vagrant location @@ -29,10 +29,9 @@ else fi cp -i /etc/kubernetes/admin.conf $config_path/config -touch $config_path/join.sh -chmod +x $config_path/join.sh kubeadm token create --print-join-command > $config_path/join.sh +chmod +x $config_path/join.sh # Install Calico Network Plugin diff --git a/scripts/node.sh b/scripts/node.sh index a575417..34a00cc 100755 --- a/scripts/node.sh +++ b/scripts/node.sh @@ -6,6 +6,11 @@ set -euxo pipefail config_path="/vagrant/configs" +while [ ! -x $config_path/join.sh ]; do + echo "Waiting for master.sh to finish..." + sleep 15 +done + /bin/bash $config_path/join.sh -v sudo -i -u vagrant bash << EOF diff --git a/settings.yaml b/settings.yaml.sample similarity index 74% rename from settings.yaml rename to settings.yaml.sample index c4b50d2..3bc0ca9 100644 --- a/settings.yaml +++ b/settings.yaml.sample @@ -1,6 +1,8 @@ --- # cluster_name is used to group the nodes in a folder within VirtualBox: cluster_name: Kubernetes Cluster +# Tell libvirt to automatically start machines when host is booting up +autostart: true # Uncomment to set environment variables for services such as crio and kubelet. # For example, configure the cluster to pull images via a proxy. # environment: | @@ -10,30 +12,32 @@ cluster_name: Kubernetes Cluster # All IPs/CIDRs should be private and allowed in /etc/vbox/networks.conf. network: # Worker IPs are simply incremented from the control IP. - control_ip: 10.0.0.10 + control_ip: 10.1.0.10 dns_servers: - 8.8.8.8 - 1.1.1.1 - pod_cidr: 172.16.1.0/16 - service_cidr: 172.17.1.0/18 + pod_cidr: 172.16.2.0/16 + service_cidr: 172.17.2.0/18 nodes: control: - cpu: 2 + cpu: 6 memory: 4096 workers: - count: 2 - cpu: 1 - memory: 2048 + count: 3 + cpu: 2 + memory: 4096 # Mount additional shared folders from the host into each virtual machine. # Note that the project directory is automatically mounted at /vagrant. # shared_folders: # - host_path: ../images # vm_path: /vagrant/images software: - box: bento/ubuntu-22.04 + box: generic/ubuntu2204 calico: 3.26.0 # To skip the dashboard installation, set its version to an empty value or comment it out: - dashboard: 2.7.0 - kubernetes: 1.29.0-* - + dashboard: + kubernetes: 1.31.1-* os: xUbuntu_22.04 + # Pick either cri-o or containerd + cri: cri-o + cri-version: 1.30.4-* \ No newline at end of file