Add compose tests

Signed-off-by: Kyle Harding <[email protected]>

Dockerfile

@@ -7,11 +7,20 @@ ARG DEBIAN_FRONTEND=noninteractive
 # hadolint ignore=DL3008
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
+    awscli \
     ca-certificates \
     curl \
     && rm -rf /var/lib/apt/lists/*
 
-RUN curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/$(uname -m)/kernels/vmlinux.bin" -O
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+# Print the available kernels in S3 in case the filenames change
+RUN aws s3 ls --no-sign-request "s3://spec.ccfc.min/firecracker-ci/v1.6/$(uname -m)/"
+
+# RUN curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/$(uname -m)/kernels/vmlinux.bin" -o vmlinux.bin
+# RUN curl -fsSL "http://mirror.archlinuxarm.org/aarch64/core/linux-aarch64-6.2.10-1-aarch64.pkg.tar.xz" -o vmlinux.bin
+# RUN curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min/img/hello/kernel/hello-vmlinux.bin" -o vmlinux.bin
+RUN curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min/firecracker-ci/v1.6/$(uname -m)/vmlinux-5.10.197" -o vmlinux.bin
 
 ###############################################
 
@@ -51,6 +60,7 @@ ARG DEBIAN_FRONTEND=noninteractive
 # hadolint ignore=DL3008
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
+    bridge-utils \
     ca-certificates \
     curl \
     e2fsprogs \
@@ -62,6 +72,7 @@ RUN apt-get update \
     jq \
     procps \
     rsync \
+    tcpdump \
     uuid-runtime \
     && rm -rf /var/lib/apt/lists/*
 
@@ -82,100 +93,60 @@ RUN chmod +x start.sh overlay/sbin/* overlay/usr/local/bin/*
 
 ENTRYPOINT [ "/usr/src/app/start.sh" ]
 
-# Default command to exec after init.
-# This should be a long-running process or service, and get overriden by the user.
-# hadolint ignore=DL3025
-CMD 'curl http://artscene.textfiles.com/asciiart/unicorn && sleep infinity'
+CMD [ "/usr/local/bin/usage.sh" ]
 
 ###############################################
 
+# Example alpine rootfs for testing, with some debug utilities
 FROM alpine:3.18 AS alpine-rootfs
 
-# WORKDIR /src
-
-# # hadolint ignore=DL3018
-# RUN apk add --no-cache openrc util-linux
+# hadolint ignore=DL3018
+RUN apk add --no-cache bash ca-certificates ca-certificates curl iproute2 iputils-ping lsblk
 
-# # Set up a login terminal on the serial console (ttyS0)
-# RUN ln -s agetty /etc/init.d/agetty.ttyS0 \
-#     && echo ttyS0 > /etc/securetty \
-#     && rc-update add agetty.ttyS0 default
+FROM jailer AS alpine-test
 
-# # Make sure special file systems are mounted on boot
-# RUN rc-update add devfs boot \
-#     && rc-update add procfs boot \
-#     && rc-update add sysfs boot
+COPY --from=alpine-rootfs / /usr/src/app/rootfs/
 
-# # Create a tarball of the root file system
-# RUN tar cf /rootfs.tar /bin /etc /lib /root /sbin /usr
+CMD "/usr/local/bin/debug.sh && sleep infinity"
 
-# hadolint ignore=DL3018
-RUN apk add --no-cache curl iproute2
+# Use livepush directives to conditionally run this test stage
+# for livepush, but not for default builds used in publishing.
+#dev-cmd-live="/usr/local/bin/debug.sh && sleep infinity"
 
 ###############################################
 
-# Use the official Ubuntu image as a base
-FROM ubuntu:jammy AS ubuntu-rootfs
-
-# # Set environment variables to avoid prompts
-# ENV DEBIAN_FRONTEND=noninteractive
-
-# # Install the necessary packages
-# # hadolint ignore=DL3008
-# RUN apt-get update \
-#     && apt-get install -y --no-install-recommends curl systemd systemd-sysv \
-#     && rm -rf /var/lib/apt/lists/*
-
-# # Remove unnecessary services
-# RUN find /etc/systemd/system \
-#         /lib/systemd/system \
-#         \( \
-#         -name "*udev*" \
-#         -o -name "*resolved*" \
-#         -o -name "*logind*" \
-#         -o -name "*getty*" \
-#         -o -name "*networkd*" \
-#         \) \
-#         -exec rm -f {} \;
-
-# # Set systemd as the entrypoint
-# STOPSIGNAL SIGRTMIN+3
-# CMD [ "/sbin/init" ]
-
-# # Set up necessary mount points
-# VOLUME [ "/sys/fs/cgroup" ]
-
-# # Copy the updated systemd service file
-# COPY entrypoint.service /etc/systemd/system/entrypoint.service
-# RUN systemctl enable entrypoint.service
-
-# COPY init /init
-# RUN chmod +x /init
+# Example debian rootfs for testing, with some debug utilities
+FROM debian:bookworm AS debian-rootfs
 
 # hadolint ignore=DL3008
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends curl iproute2 \
+    && apt-get install -y --no-install-recommends curl iproute2 iputils-ping ca-certificates util-linux \
     && rm -rf /var/lib/apt/lists/*
 
+FROM jailer AS debian-test
+
+COPY --from=debian-rootfs / /usr/src/app/rootfs/
+
+CMD "/usr/local/bin/debug.sh && sleep infinity"
+
 ###############################################
 
-FROM ghcr.io/product-os/self-hosted-runners:v3.3.3 AS self-hosted-runners
+# Example ubuntu rootfs for testing, with some debug utilities
+FROM ubuntu:jammy AS ubuntu-rootfs
 
 # hadolint ignore=DL3008
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends curl iproute2 \
+    && apt-get install -y --no-install-recommends ca-certificates curl iproute2 iputils-ping util-linux \
     && rm -rf /var/lib/apt/lists/*
 
-CMD [ "/init" ]
+FROM jailer AS ubuntu-test
 
-###############################################
+COPY --from=ubuntu-rootfs / /usr/src/app/rootfs/
 
-# Include firecracker wrapper and scripts
-FROM jailer AS runtime
+CMD "/usr/local/bin/debug.sh && sleep infinity"
 
-# Copy the root file system from your container final stage
-COPY --from=alpine-rootfs / /usr/src/app/rootfs/
-# COPY --from=ubuntu-rootfs / /usr/src/app/rootfs/
-# COPY --from=self-hosted-runners / /usr/src/app/rootfs/
+###############################################
 
-CMD 'curl http://artscene.textfiles.com/asciiart/unicorn ; echo $SECRET_KEY ; sleep infinity'
+# This is the stage we want to publish, but it has no rootfs
+# so we can't use it for livepush testing.
+FROM jailer AS default

config.json

@@ -39,5 +39,11 @@
   "logger": null,
   "metrics": null,
   "mmds-config": null,
-  "entropy": null
+  "rate_limiter": {
+    "bandwidth": {
+      "size": 1000,
+      "one_time_burst": 0,
+      "refill_time": 100
+    }
+  }
 }

docker-compose.test.yml

@@ -2,7 +2,74 @@ version: "2.4"
 
 services:
   sut:
+    image: alpine:3.18
+    depends_on:
+      - alpine-test
+      - debian-test
+      - ubuntu-test
+    volumes:
+      - alpine-data:/alpine-data
+      - debian-data:/debian-data
+      - ubuntu-data:/ubuntu-data
+    command:
+      - /bin/sh
+      - -c
+      - |
+        for vol in /alpine-data /debian-data /ubuntu-data; do
+
+          count=0
+          while true; do
+            if [ $$count -gt 10 ]; then
+              echo "Failed to find $$vol/datafs.ext4"
+              exit 1
+            fi
+            if [ -f $$vol/datafs.ext4 ]; then  
+              break
+            fi
+            sleep 1
+            count=$$((count + 1))
+          done
+
+          mkdir -p /mnt/$$vol
+          mount $$vol/datafs.ext4 /mnt/$$vol
+          test -f /mnt/$$vol/ok
+
+        done
+
+  alpine-test:
     extends:
       file: docker-compose.yml
       service: firecracker
-    image: localhost:5000/sut
+    build:
+      context: .
+      target: alpine-test
+    volumes:
+      - alpine-data:/jail/data
+    command: /usr/local/bin/debug.sh
+
+  debian-test:
+    extends:
+      file: docker-compose.yml
+      service: firecracker
+    build:
+      context: .
+      target: debian-test
+    volumes:
+      - debian-data:/jail/data
+    command: /usr/local/bin/debug.sh
+
+  ubuntu-test:
+    extends:
+      file: docker-compose.yml
+      service: firecracker
+    build:
+      context: .
+      target: ubuntu-test
+    volumes:
+      - ubuntu-data:/jail/data
+    command: /usr/local/bin/debug.sh
+
+volumes:
+  alpine-data: {}
+  debian-data: {}
+  ubuntu-data: {}

docker-compose.yml

@@ -7,14 +7,10 @@ services:
     # but permissions are dropped to a chroot in order to start your VM
     privileged: true
     network_mode: host
-    # Optionally run the VM rootfs and kernel in-memory to save storage wear
+    # Optionally run the VM jail in-memory to save storage wear
     tmpfs:
       - /tmp
       - /run
       - /srv
-    volumes:
-      - persistent-data:/jail/data
-
-volumes:
-  persistent-data: {}
-
+    environment:
+      - CTR_SECRET_KEY=secretvalue

overlay/sbin/init

@@ -4,20 +4,52 @@ exec 1>/dev/console
 exec 2>/dev/console
 
 # Mount essential file systems
-mount -t proc none /proc
-mount -t sysfs none /sys
-mount -t devtmpfs none /dev
+if ! mountpoint -q /proc; then
+  mount -t proc none /proc
+fi
 
-# Mount tmpfs
-mount -o remount,rw,exec tmpfs /tmp
-mount -o remount,rw,exec tmpfs /var/run
+if ! mountpoint -q /sys; then
+  mount -t sysfs none /sys
+fi
 
-# Bring up networking
+if ! mountpoint -q /dev; then
+  mount -t devtmpfs none /dev
+fi
+
+if ! mountpoint -q /tmp; then
+  mount -t tmpfs none /tmp
+fi
+
+if ! mountpoint -q /run; then
+  mount -t tmpfs none /run
+fi
+
+# rngd -b
+
+# The IP is assigned by converting the last 4 hexa groups of the MAC into decimals.
 # https://github.com/firecracker-microvm/firecracker/blob/main/resources/overlay/usr/local/bin/fcnet-setup.sh
-/usr/local/bin/fcnet-setup.sh
+for dev in $(ip link list | awk /'^[0-9]+:/ {print $2}' | sed 's/://'); do
+    dev="$(basename "$dev")"
+    case $dev in
+    *lo) continue ;;
+    esac
+    for octet in $(
+        ip link show dev "$dev" |
+            awk '/link\/ether/ {print $2}' |
+            awk -F: '{print $3" "$4" "$5" "$6}'
+    ); do
+        ip=$ip$(printf "%d" 0x"$octet").
+    done
+    ip=${ip%?}
+    ip addr add "$ip/30" dev "$dev"
+    ip link set "$dev" up
+    ip route add default via "${ip%?}1" dev "$dev"
+done
 
 # Export secrets to the environment and remove the files
-for f in /var/secrets/*; do
-    eval "export $(basename "${f}")=$(cat "${f}")"
-    rm -f "${f}"
-done
+if [ "$(ls /var/secrets)" ]; then
+    for f in /var/secrets/*; do
+        eval "export $(basename "${f}")=$(cat "${f}")"
+        rm -f "${f}"
+    done
+fi

overlay/usr/local/bin/debug.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+set -ex
+
+date
+
+uname -a
+
+df -h
+lsblk
+
+ip link list
+ip route
+
+ping -c 4 "$(ip route | awk '/default/ {print $3}')"
+ping -c 4 "$(head -1 /etc/resolv.conf | awk '{print $2}')"
+ping -c 4 -M "do" -s 1472 "$(head -1 /etc/resolv.conf | awk '{print $2}')"
+
+curl -fsSL https://raw.githubusercontent.com/dylanaraps/neofetch/7.1.0/neofetch | bash
+
+test -n "${SECRET_KEY}"
+
+mkdir -p /mnt/data
+mount -v /dev/vdb /mnt/data
+touch /mnt/data/ok