From e32fe55c33f68d62af1a356be26d8aa4027bb2c3 Mon Sep 17 00:00:00 2001 From: Dimitris Karakasilis Date: Fri, 17 Jan 2025 09:26:24 +0200 Subject: [PATCH] [WIP] Create test pipeline Signed-off-by: Dimitris Karakasilis --- .github/release-space-from-ubuntu-runners.sh | 46 ++++ .github/upload-image-to-aws.sh | 235 +++++++++++++++++++ .github/workflows/push-aws-test.yaml | 97 ++++++++ .github/workflows/release.yaml | 207 +++++++--------- 4 files changed, 464 insertions(+), 121 deletions(-) create mode 100644 .github/release-space-from-ubuntu-runners.sh create mode 100755 .github/upload-image-to-aws.sh create mode 100644 .github/workflows/push-aws-test.yaml diff --git a/.github/release-space-from-ubuntu-runners.sh b/.github/release-space-from-ubuntu-runners.sh new file mode 100644 index 000000000..0eefb5bc8 --- /dev/null +++ b/.github/release-space-from-ubuntu-runners.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +echo "Listing top largest packages" +pkgs=$(dpkg-query -Wf '${Installed-Size}\t${Package}\t${Status}\n' | awk '$NF == "installed"{print $1 "\t" $2}' | sort -nr) +head -n 30 <<< "${pkgs}" +echo +df -h +echo +sudo apt-get remove -y '^llvm-.*|^libllvm.*' || true +sudo apt-get remove --auto-remove android-sdk-platform-tools || true +sudo apt-get purge --auto-remove android-sdk-platform-tools || true +sudo rm -rf /usr/local/lib/android +sudo apt-get remove -y '^dotnet-.*|^aspnetcore-.*' || true +sudo rm -rf /usr/share/dotnet +sudo apt-get remove -y '^mono-.*' || true +sudo apt-get remove -y '^ghc-.*' || true +sudo apt-get remove -y '.*jdk.*|.*jre.*' || true +sudo apt-get remove -y 'php.*' || true +sudo apt-get remove -y hhvm || true +sudo apt-get remove -y powershell || true +sudo apt-get remove -y firefox || true +sudo apt-get remove -y monodoc-manual || true +sudo apt-get remove -y msbuild || true +sudo apt-get remove -y microsoft-edge-stable || true +sudo apt-get remove -y '^google-.*' || true +sudo apt-get remove -y azure-cli || true +sudo apt-get remove -y '^mongo.*-.*|^postgresql-.*|^mysql-.*|^mssql-.*' || true +sudo apt-get remove -y '^gfortran-.*' || true +sudo apt-get remove -y '^gcc-*' || true +sudo apt-get remove -y '^g++-*' || true +sudo apt-get remove -y '^cpp-*' || true +sudo apt-get autoremove -y +sudo apt-get clean +echo +echo "Listing top largest packages" +pkgs=$(dpkg-query -Wf '${Installed-Size}\t${Package}\t${Status}\n' | awk '$NF == "installed"{print $1 "\t" $2}' | sort -nr) +head -n 30 <<< "${pkgs}" +echo +sudo rm -rfv build || true + +sudo rm -rf /usr/local/lib/android # will release about 10 GB if you don't need Android +sudo rm -rf /usr/share/dotnet # will release about 20GB if you don't need .NET + +df -h + + diff --git a/.github/upload-image-to-aws.sh b/.github/upload-image-to-aws.sh new file mode 100755 index 000000000..c8bf68b4f --- /dev/null +++ b/.github/upload-image-to-aws.sh @@ -0,0 +1,235 @@ +#!/bin/bash + +# Given a raw image created with Auroraboot, this script will upload it to the speficied AWS account as a public AMI. +# Docs: +# https://docs.aws.amazon.com/vm-import/latest/userguide/required-permissions.html +# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/creating-an-ami-ebs.html#creating-launching-ami-from-snapshot +# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-ami-boot-mode.html +# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/launch-instance-boot-mode.html + +set -e +set -o pipefail + +checkArguments() { + if [ $# -lt 1 ]; then + echo "Error: You need to specify the cloud image to upload." + echo "Usage: $0 " + exit 1 + fi + + local file="$1" + + if [ ! -f "$file" ]; then + echo "Error: File '$file' does not exist." + exit 1 + fi + + if ! file "$file" | grep -q 'DOS/MBR boot sector'; then + echo "Error: File '$file' is not a raw image." + exit 1 + fi +} + +checkEnvVars() { + if [ -z "$AWS_PROFILE" ] || [ -z "$AWS_REGION" ] || [ -z "$AWS_S3_BUCKET" ]; then + echo "Error: AWS_PROFILE, AWS_REGION and AWS_S3_BUCKET environment variables must be set." + exit 1 + fi +} + +AWS() { + aws --profile $AWS_PROFILE --region $AWS_REGION "$@" +} + +# https://docs.aws.amazon.com/vm-import/latest/userguide/required-permissions.html#vmimport-role +ensureVmImportRole() { + (AWS iam list-roles | jq -r '.Roles[] | select(.RoleName | contains("vmimport")) | .RoleName' | grep -q "vmimport" && echo "vmimport role found. All good.") || { + echo "Creating vmimport role" + + export AWS_PAGER="" # Avoid being dropped to a pager + AWS iam create-role --role-name vmimport --assume-role-policy-document file://<(cat < /dev/null 2>&1; then + echo "File '$baseName' already exists in S3 bucket '$AWS_S3_BUCKET'." + else + echo "File '$baseName' does not exist in S3 bucket '$AWS_S3_BUCKET'. Uploading now." + AWS s3 cp $1 s3://$AWS_S3_BUCKET/$baseName + fi +} + +waitForSnapshotCompletion() { + local taskID="$1" + local status="" + + while true; do + status=$(AWS ec2 describe-import-snapshot-tasks --import-task-ids "$taskID" --query 'ImportSnapshotTasks[0].SnapshotTaskDetail.Status' --output text) + + if [ "$status" == "completed" ]; then + echo "Snapshot import completed." + break + elif [ "$status" == "deleted" ] || [ "$status" == "cancelling" ] || [ "$status" == "cancelled" ]; then + echo "Snapshot import failed with status: $status" + exit 1 + else + echo "Waiting for snapshot import to complete. Current status: $status" >&2 + sleep 30 + fi + done + + echo $(AWS ec2 describe-import-snapshot-tasks --import-task-ids "$taskID" --query 'ImportSnapshotTasks[0].SnapshotTaskDetail.SnapshotId' --output text) +} + +importAsSnapshot() { + local file="$1" + local snapshotID + + snapshotID=$(AWS ec2 describe-snapshots --filters "Name=tag:SourceFile,Values=$file" --query "Snapshots[0].SnapshotId" --output text) + if [ "$snapshotID" != "None" ]; then + echo "Snapshot $snapshotID already exists for file $file" + echo $snapshotID + return 0 + fi + + taskID=$(AWS ec2 import-snapshot --description "$file" --disk-container file://<(cat <