wip #1

Workflow file for this run

	---
	name: openBalena tests

	on:
	workflow_call:

	# https://docs.github.com/en/actions/security-guides/automatic-token-authentication
	# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions
	permissions:
	contents: read
	id-token: write # AWS GitHub OIDC required: write
	packages: read

	env:
	# Stack ID
	# arn:aws:cloudformation:us-east-1:491725000532:stack/balena-tests-s3-certs/814dea60-404d-11ed-b06f-0a7d458f8ba5
	AWS_S3_CERTS_BUCKET: balena-tests-certs
	# (kvm) nested virtualisation not supported on AWS/EC2 instance types\|classes other than X.metal
	AWS_EC2_INSTANCE_TYPES: "c6a.xlarge c6i.xlarge c5n.xlarge c5.xlarge c5a.xlarge m5.xlarge m5n.xlarge m5a.xlarge"
	AWS_EC2_LAUNCH_TEMPLATE: lt-02e10a4f66261319d
	AWS_EC2_LT_VERSION: "5"
	AWS_IAM_USERNAME: balena-tests-iam-User-1GXO3XP12N6LL
	AWS_LOGS_RETENTION: "30"
	AWS_VPC_SECURITY_GROUP_IDS: sg-057937f4d89d9d51c
	AWS_VPC_SUBNET_IDS: 'subnet-02d18a08ea4058574 subnet-0a026eae1df907a09'
	# otherwise it tries to send data to an endpoint provided by a private project
	# https://github.com/balena-io/analytics-backend
	# .. which is not part of openBalena
	BALENARC_NO_ANALYTICS: "1" # https://github.com/balena-io/balena-cli/blob/master/lib/events.ts#L62-L70
	DEBUG: "0" # https://github.com/balena-io/balena-cli/issues/2447
	RETRY: "3"

	jobs:
Check failure on line 33 in .github/workflows/tests.yml View workflow run for this annotation GitHub Actions / .github/workflows/tests.yml Invalid workflow file `You have an error in your yaml syntax on line 33`
	test:
	runs-on: ["self-hosted", "X64", "distro:jammy"] # tests require socat v1.7.4
	timeout-minutes: 60
	strategy:
	matrix:
	target:
	- compose-private-pki
	- balena-public-pki
	include:
	# tests compose flow using self-signed PKI
	- target: compose-private-pki
	# Canonical, Ubuntu, 24.04 LTS, amd64 noble image build on 2024-04-23
	ami: ami-04b70fa74e45c3917
	subdomain: auto
	dns_tld: balena-devices.com
	verbose: ${{ vars.VERBOSE \|\| 'false' }}

	# .. balena flow using Let's Encrypt (ACME) PKI
	# https://dash.cloudflare.com/001b3ed2352612aaa068aca1b0022736/balena-devices.com/dns
	- target: balena-public-pki
	# balenaOS-2.113.12-generic-amd64
	ami: ami-03a3995797dee84fa
	subdomain: auto
	dns_tld: balena-devices.com
	environment: balena-cloud.com
	fleet: balena/open-balena
	verbose: ${{ vars.VERBOSE \|\| 'false' }}

	# https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#handling-failures
	fail-fast: true
	environment:
	name: ${{ matrix.target }}

	steps:
	- uses: actions/checkout@b80ff79f1755d06ba70441c368a6fe801f5f3a62

	# https://github.com/unfor19/install-aws-cli-action
	- name: Setup awscli
	uses: unfor19/install-aws-cli-action@v1

	- uses: aws-actions/configure-aws-credentials@bd0758102444af2a09b9e47a2c93d0f091c1252d
	with:
	aws-region: ${{ vars.AWS_REGION \|\| 'us-east-1' }}
	role-session-name: github-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }}
	# balena-io/environments-bases: aws/balenacloud/ephemeral-tests/balena-tests-iam.yml
	role-to-assume: ${{ vars.AWS_IAM_ROLE }}

	# https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager-working-with-install-plugin.html#install-plugin-debian
	- name: install session-manager-plugin
	if: \|
	github.event_name == 'pull_request' &&
	github.event.action != 'closed' &&
	matrix.target == 'compose-private-pki'
	run: \|
	runner_arch="$(echo "${RUNNER_ARCH}" \| tr '[:upper:]' '[:lower:]' \| sed 's/x64/64bit/g')"

	session-manager-plugin \|\| (curl -sSfo session-manager-plugin.deb https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_${runner_arch}/session-manager-plugin.deb \
	&& sudo dpkg -i session-manager-plugin.deb \
	&& rm -f session-manager-plugin.deb)

	# https://github.com/balena-io-examples/setup-balena-action
	- name: Setup balena CLI
	uses: balena-io-examples/setup-balena-action@main

	# https://github.com/pdcastro/ssh-uuid#why
	# https://github.com/pdcastro/ssh-uuid#linux-debian-ubuntu-others
	- name: install additional dependencies
	if: \|
	github.event_name == 'pull_request' &&
	github.event.action != 'closed' &&
	matrix.target == 'balena-public-pki'
	shell: bash
	run: \|
	set -ue
	echo '::notice::install additional dependencies'

	[[ '${{ matrix.verbose \|\| 'false' }}' =~ on\|On\|Yes\|yes\|true\|True ]] && set -x

	source src/balena-tests/functions

	mkdir -p "${RUNNER_TEMP}/ssh-uuid"

	wget -q -O "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" https://raw.githubusercontent.com/pdcastro/ssh-uuid/master/ssh-uuid.sh \
	&& chmod +x "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" \
	&& ln -s "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" "${RUNNER_TEMP}/ssh-uuid/scp-uuid"

	with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'

	balena version

	"${RUNNER_TEMP}/ssh-uuid/scp-uuid" --help

	grep -q "${RUNNER_TEMP}/ssh-uuid" "${GITHUB_PATH}" \
	\|\| echo "${RUNNER_TEMP}/ssh-uuid" >> "${GITHUB_PATH}"

	- name: generate SSH key
	id: generate-key-pair
	if: \|
	github.event_name == 'pull_request' &&
	github.event.action != 'closed'
	run: \|
	set -ue

	verbose='+x'
	if [[ '${{ matrix.verbose }}' =~ on\|On\|Yes\|yes\|true\|True ]]; then
	verbose='-x'
	fi
	set ${verbose}

	key_name="${{ matrix.target }}-${GITHUB_RUN_ID}-${GITHUB_RUN_NUMBER}-${GITHUB_RUN_ATTEMPT}"
	echo "key_name=${key_name}" >> $GITHUB_OUTPUT

	set +x
	private_key_material="$(aws ec2 create-key-pair \
	--key-name "${key_name}" \| jq -r .KeyMaterial)"

	public_key="$(aws ec2 describe-key-pairs --include-public-key \
	--key-name "${key_name}" \| jq -re .KeyPairs[].PublicKey)"

	# https://stackoverflow.com/a/70384422/1559300
	# https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#masking-a-value-in-log
	while read -r line; do
	echo "::add-mask::${line}"
	done <<< "${private_key_material}"

	ssh_private_key="$(cat << EOF
	$(echo "${private_key_material}")
	EOF
	)"
	echo "ssh_private_key<<EOF" >> $GITHUB_OUTPUT
	set ${verbose}

	echo "${ssh_private_key}" >> $GITHUB_OUTPUT
	echo "EOF" >> $GITHUB_OUTPUT

	echo "${ssh_public_key}" >> $GITHUB_OUTPUT

	env:
	AWS_DEFAULT_REGION: ${{ vars.AWS_REGION \|\| 'us-east-1' }}

	# https://github.com/webfactory/ssh-agent
	- uses: webfactory/ssh-agent@dc588b651fe13675774614f8e6a936a468676387 # v0.9.0
	if: \|
	github.event_name == 'pull_request' &&
	github.event.action != 'closed'
	with:
	ssh-private-key: ${{ steps.generate-key-pair.outputs.ssh_private_key }}

	- name: provision SSH key
	id: provision-ssh-key
	# wait for cloud-config
	# https://github.com/balena-os/cloud-config
	timeout-minutes: 5
	if: \|
	github.event_name == 'pull_request' &&
	github.event.action != 'closed' &&
	matrix.target == 'balena-public-pki'
	run: \|
	set -ue

	[[ '${{ matrix.verbose \|\| 'false' }}' =~ on\|On\|Yes\|yes\|true\|True ]] && set -x

	source src/balena-tests/functions

	with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'

	if ! [[ -e "${HOME}/.ssh/id_rsa" ]]; then
	echo '${{ steps.generate-key-pair.outputs.ssh_private_key }}' > "${HOME}/.ssh/id_rsa"
	echo '${{ steps.generate-key-pair.outputs.ssh_public_key }}' > "${HOME}/.ssh/id_rsa.pub"
	fi

	echo "::notice::check $(balena keys \| wc -l) keys"

	match=''
	for key in $(balena keys \| grep -v ID \| awk '{print $1}'); do
	fp=$(balena key ${key} \| tail -n 1 \| ssh-keygen -E md5 -lf /dev/stdin \| awk '{print $2}')
	if [[ $fp =~ $(ssh-keygen -E md5 -lf "${HOME}/.ssh/id_rsa" \| awk '{print $2}') ]]; then
	match="${key}"
	break
	fi
	done

	if [[ -z $match ]]; then
	balena key add "${GITHUB_SHA}" "${HOME}/.ssh/id_rsa.pub"
	else
	balena keys
	fi

	while ! [[ "$(ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
	${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
	cat /mnt/boot/config.json \| jq -r .uuid)" =~ ${{ steps.register-test-device.outputs.balena_device_uuid }} ]]; do

	echo "::warning::Still working..."
	sleep "$(( (RANDOM % 5) + 5 ))s"
	done

	echo "key_id=${GITHUB_SHA}" >> "${GITHUB_OUTPUT}"

	- name: (pre)register test device
	id: register-test-device
	if: \|
	github.event_name == 'pull_request' &&
	github.event.action != 'closed' &&
	matrix.target == 'balena-public-pki'
	run: \|
	set -ue

	[[ '${{ matrix.verbose \|\| 'false' }}' =~ on\|On\|Yes\|yes\|true\|True ]] && set -x

	source src/balena-tests/functions

	with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'

	balena_device_uuid="$(openssl rand -hex 16)"

	# https://www.balena.io/docs/learn/more/masterclasses/advanced-cli/#52-preregistering-a-device
	with_backoff balena device register '${{ matrix.fleet }}' --uuid "${balena_device_uuid}"

	device_id="$(balena device "${balena_device_uuid}" \| grep ^ID: \| cut -c20-)"

	# the actual version deployed depends on the AWS EC2/AMI, defined in AWS_EC2_LAUNCH_TEMPLATE
	os_version="$(balena os versions ${{ vars.DEVICE_TYPE \|\| 'generic-amd64' }} \| head -n 1)"

	balena config generate \
	--version "${os_version}" \
	--device "${balena_device_uuid}" \
	--network ethernet \
	--appUpdatePollInterval 10 \
	$([[ '${{ vars.DEVELOPMENT_MODE \|\| 'false' }}' =~ true ]] && echo '--dev') \
	--output config.json

	with_backoff balena tag set balena ephemeral-test-device --device "${balena_device_uuid}"

	github_vars=(GITHUB_ACTOR GITHUB_BASE_REF GITHUB_HEAD_REF GITHUB_JOB \
	GITHUB_REF GITHUB_REF_NAME GITHUB_REF_TYPE GITHUB_REPOSITORY \
	GITHUB_REPOSITORY_OWNER GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_RUN_NUMBER \
	GITHUB_SHA GITHUB_WORKFLOW RUNNER_ARCH RUNNER_NAME RUNNER_OS)

	for github_var in "${github_vars[@]}"; do
	balena tag set ${github_var} "${!github_var}" --device "${balena_device_uuid}"
	done

	echo "balena_device_uuid=${balena_device_uuid}" >> "${GITHUB_OUTPUT}"
	echo "balena_device_id=${device_id}" >> "${GITHUB_OUTPUT}"

	# https://github.com/balena-io/balena-cli/issues/1543
	- name: pin device to draft release
	if: \|
	github.event_name == 'pull_request' &&
	github.event.action != 'closed' &&
	matrix.target == 'balena-public-pki'
	run: \|
	set -uae

	[[ '${{ matrix.verbose \|\| 'false' }}' =~ on\|On\|Yes\|yes\|true\|True ]] && set -x

	source src/balena-tests/functions

	with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'

	pr_id='${{ github.event.pull_request.id }}'
	head_sha='${{ github.event.pull_request.head.sha \|\| github.event.head_commit.id }}'
	release_id="$(with_backoff balena releases '${{ matrix.fleet }}' --json \
	\| jq -r --arg pr_id "${pr_id}" --arg head_sha "${head_sha}" '.[]
	\| select(.release_tag[].tag_key=="balena-ci-commit-sha")
	\| select(.release_tag[].value==$head_sha)
	\| select(.release_tag[].tag_key=="balena-ci-id")
	\| select(.release_tag[].value==$pr_id).commit')"

	with_backoff balena device pin \
	${{ steps.register-test-device.outputs.balena_device_uuid }} \
	"${release_id}"

	with_backoff balena device ${{ steps.register-test-device.outputs.balena_device_uuid }}

	- name: configure test device environment
	if: \|
	github.event_name == 'pull_request' &&
	github.event.action != 'closed' &&
	matrix.target == 'balena-public-pki'
	run: \|
	set -ue

	[[ '${{ matrix.verbose \|\| 'false' }}' =~ on\|On\|Yes\|yes\|true\|True ]] && set -x

	source src/balena-tests/functions

	with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'

	with_backoff balena env add VERBOSE "${{ vars.VERBOSE \|\| 'false' }}" \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	with_backoff balena env add BALENARC_NO_ANALYTICS '1' \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	with_backoff balena env add DNS_TLD '${{ matrix.target }}' \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	with_backoff balena env add DB_HOST db \
	--service api \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	with_backoff balena env add REDIS_HOST redis:6379 \
	--service api \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'


	# to allow devices running locally to communicate to the local API, we can route
	# to the local Docker network aliases instead of public DNS, since (a) DNS_TLD is
	# guestfwd(ed) in QEMU to a special internal IP 10.0.2.100; (b) is proxied to
	# haproxy network alias on device; and (c) made public with a wildcard DNS record
	# (e.g.)
	#
	# $ dig +short api.auto.balena-devices.com
	# 10.0.2.100
	#

	with_backoff balena env add API_HOST 'api.${{ matrix.target }}' \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	# not used but required for config.json to be valid
	with_backoff balena env add DELTA_HOST 'delta.${{ matrix.target }}' \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	with_backoff balena env add REGISTRY2_HOST 'registry2.${{ matrix.target }}' \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	with_backoff balena env add VPN_HOST 'cloudlink.${{ matrix.target }}' \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	with_backoff balena env add HOST 'api.${{ matrix.target }}' \
	--service api \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	with_backoff balena env add TOKEN_AUTH_CERT_ISSUER 'api.${{ matrix.target }}' \
	--service api \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	with_backoff balena env add REGISTRY2_TOKEN_AUTH_ISSUER 'api.${{ matrix.target }}' \
	--service registry \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	with_backoff balena env add REGISTRY2_TOKEN_AUTH_REALM 'https://api.${{ matrix.target }}/auth/v1/token' \
	--service registry \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	with_backoff balena env add REGISTRY2_S3_REGION_ENDPOINT 's3.${{ matrix.target }}' \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	with_backoff balena env add WEBRESOURCES_S3_HOST 's3.${{ matrix.target }}' \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	# https://github.com/balena-io/cert-manager/blob/master/entry.sh#L255-L278
	# cert-manager will restore the last wildcard certificate from AWS/S3 to avoid
	# being rate limited by LetsEncrypt/ACME
	with_backoff balena env add AWS_S3_BUCKET '${{ env.AWS_S3_CERTS_BUCKET }}' \
	--service cert-manager \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	# FIXME: still required?
	with_backoff balena env add COMMON_REGION '${{ env.AWS_REGION }}' \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	with_backoff balena env add SUPERUSER_EMAIL 'admin@${{ matrix.target }}' \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	with_backoff balena env add ORG_UNIT openBalena \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'


	# unstable/unsupported functionality
	with_backoff balena env add HIDE_UNVERSIONED_ENDPOINT 'false' \
	--service api \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	with_backoff balena env add RELEASE_ASSETS_TEST 'true' \
	--service sut \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	- name: configure test device secrets
	if: \|
	github.event_name == 'pull_request' &&
	github.event.action != 'closed' &&
	matrix.target == 'balena-public-pki'
	run: \|
	set -ue

	[[ '${{ matrix.verbose \|\| 'false' }}' =~ on\|On\|Yes\|yes\|true\|True ]] && set -x

	source src/balena-tests/functions

	with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'

	# cert-manager requires it to get whoami information for the user
	with_backoff balena env add API_TOKEN '${{ secrets.BALENA_API_KEY }}' \
	--service cert-manager \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	# cert-manager requires is to request wildcard SSL certificate from LetsEncrypt
	with_backoff balena env add CLOUDFLARE_API_TOKEN '${{ secrets.CLOUDFLARE_API_TOKEN }}' \
	--service cert-manager \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	# AWS credentials to backup/restore PKI assets
	with_backoff balena env add AWS_ACCESS_KEY_ID '${{ env.AWS_ACCESS_KEY_ID }}' \
	--service cert-manager \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	with_backoff balena env add AWS_SECRET_ACCESS_KEY '${{ env.AWS_SECRET_ACCESS_KEY }}' \
	--service cert-manager \
	--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'

	- name: provision balenaOS ephemeral SUT
	id: balena-sut
	if: \|
	github.event_name == 'pull_request' &&
	github.event.action != 'closed' &&
	matrix.target == 'balena-public-pki'
	run: \|
	set -ue

	[[ '${{ matrix.verbose \|\| 'false' }}' =~ on\|On\|Yes\|yes\|true\|True ]] && set -x

	for subnet_id in ${{ env.AWS_VPC_SUBNET_IDS }}; do
	# spot, on-demand
	for market_type in ${{ vars.MARKET_TYPES \|\| 'spot' }}; do
	for instance_type in ${AWS_EC2_INSTANCE_TYPES}; do
	# https://docs.aws.amazon.com/cli/latest/reference/ec2/run-instances.html
	response="$(aws ec2 run-instances \
	--image-id ${{ matrix.ami }} \
	--launch-template 'LaunchTemplateId=${{ env.AWS_EC2_LAUNCH_TEMPLATE }},Version=${{ env.AWS_EC2_LT_VERSION }}' \
	--instance-type "${instance_type}" \
	$([[ $market_type =~ spot ]] && echo '--instance-market-options MarketType=spot') \
	--security-group-ids '${{ env.AWS_VPC_SECURITY_GROUP_IDS }}' \
	--subnet-id "${subnet_id}" \
	--associate-public-ip-address \
	--user-data file://config.json \
	--tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=open-balena-tests},{Key=matrix.target,Value=${{ matrix.target }}},{Key=MarketType,Value=${market_type}},{Key=Owner,Value=${{ env.AWS_IAM_USERNAME }}},{Key=GITHUB_SHA,Value=${GITHUB_SHA}-tests},{Key=GITHUB_RUN_ID,Value=${GITHUB_RUN_ID}-tests},{Key=GITHUB_RUN_NUMBER,Value=${GITHUB_RUN_NUMBER}-tests},{Key=GITHUB_RUN_ATTEMPT,Value=${GITHUB_RUN_ATTEMPT}-tests}]" \|\| true)"

	[[ -n $response ]] && break
	done
	[[ -n $response ]] && break
	done
	[[ -n $response ]] && break
	done

	[[ -z $response ]] && exit 1

	instance_id="$(echo "${response}" \| jq -r '.Instances[].InstanceId')"

	aws ec2 wait instance-running --instance-ids "${instance_id}"

	aws ec2 wait instance-status-ok --instance-ids "${instance_id}"

	echo "instance_id=${instance_id}" >> "${GITHUB_OUTPUT}"

	env:
	AWS_DEFAULT_REGION: ${{ vars.AWS_REGION \|\| 'us-east-1' }}

	- name: wait for application
	timeout-minutes: 10
	if: \|
	github.event_name == 'pull_request' &&
	github.event.action != 'closed' &&
	matrix.target == 'balena-public-pki'
	run: \|
	set -ue

	[[ '${{ matrix.verbose \|\| 'false' }}' =~ on\|On\|Yes\|yes\|true\|True ]] && set -x

	source src/balena-tests/functions

	with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'

	balena whoami && ssh-add -l

	while [[ "$(curl -X POST --silent --retry ${{ env.RETRY }} --fail \
	'https://api.${{ matrix.environment }}/supervisor/v1/device' \
	--header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \
	--header 'Content-Type:application/json' \
	--data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \
	--compressed \| jq -r '.update_pending')" =~ ^true$ ]]; do

	sleep "$(( ( RANDOM % ${{ env.RETRY }} ) + ${{ env.RETRY }} ))s"
	done

	# wait for services to start running
	while with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
	${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
	'balena ps -q \| xargs balena inspect \| jq -r .[].State.Status' \
	\| grep -E 'created\|restarting\|removing\|paused\|exited\|dead'; do

	echo "::warning::Still working..."
	sleep "$(( (RANDOM % 30) + 30 ))s"
	done

	# wait for Docker healthchecks
	while with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
	${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
	'balena ps -q \| xargs balena inspect \
	\| jq -r ".[] \| select(.State.Health.Status!=null).Name + \":\" + .State.Health.Status"' \
	\| grep -E ':starting\|:unhealthy'; do

	echo "::warning::Still working..."
	sleep "$(( (RANDOM % 30) + 30 ))s"
	done

	# (TBC) https://www.balena.io/docs/reference/supervisor/docker-compose/
	# due to lack of long form depends_on support in compositions, restart to ensure all
	# components are running with the latest configuration; preferred over restart via
	# Supervisor API restart due to potential HTTP [timeouts](https://github.com/balena-os/balena-supervisor/issues/1157)
	- name: restart components
	timeout-minutes: 10
	if: \|
	github.event_name == 'pull_request' &&
	github.event.action != 'closed' &&
	matrix.target == 'balena-public-pki'
	run: \|
	set -ue

	[[ '${{ matrix.verbose \|\| 'false' }}' =~ on\|On\|Yes\|yes\|true\|True ]] && set -x

	source src/balena-tests/functions

	with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'

	balena whoami && ssh-add -l

	with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
	${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
	"balena ps -aq \| xargs balena inspect \
	\| jq -re '.[]
	\| select(.Name \| contains(\"_supervisor\") \| not).Id' \
	\| xargs balena restart"

	# wait for Docker healthchecks
	while with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
	${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
	'balena ps -q \| xargs balena inspect \
	\| jq -r ".[] \| select(.State.Health.Status!=null).Name + \":\" + .State.Health.Status"' \
	\| grep -E ':starting\|:unhealthy'; do

	echo "::warning::Still working..."
	sleep "$(( (RANDOM % 30) + 30 ))s"
	done

	- name: SUT&DUT (balena)
	if: \|
	github.event_name == 'pull_request' &&
	github.event.action != 'closed' &&
	matrix.target == 'balena-public-pki'
	timeout-minutes: 20
	# https://giters.com/gfx/example-github-actions-with-tty
	# https://github.com/actions/runner/issues/241#issuecomment-924327172
	shell: 'script -q -e -c "bash {0}"'
	run: \|
	set -ue

	[[ '${{ matrix.verbose \|\| 'false' }}' =~ on\|On\|Yes\|yes\|true\|True ]] && set -x

	source src/balena-tests/functions

	with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'

	balena whoami && ssh-add -l

	(with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
	${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
	"balena ps -aq \| xargs balena inspect \
	\| jq -re '.[] \| select(.Name \| contains(\"sut_\")).Id' \
	\| xargs balena logs -f") &

	# tests service is working while its status == running
	status=''
	while [[ "$status" =~ Running ]]; do
	status="$(curl --silent --retry ${{ env.RETRY }} --fail \
	'https://api.${{ matrix.environment }}/supervisor/v2/applications/state' \
	--header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \
	--header 'Content-Type:application/json' \
	--data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \
	--compressed \| jq -r '.[].services.sut.status')"

	echo "::warning::Still working..."
	sleep "$(( ( RANDOM % ${{ env.RETRY }} ) + ${{ env.RETRY }} ))s"
	done

	# .. once the service exits with status == exited, it is assumed to be finished
	status=''
	while ! [[ "$status" =~ exited ]]; do
	echo "::warning::Still working..."
	status="$(curl --silent --retry ${{ env.RETRY }} --fail \
	'https://api.${{ matrix.environment }}/supervisor/v2/applications/state' \
	--header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \
	--header 'Content-Type:application/json' \
	--data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \
	--compressed \| jq -r '.[].services.sut.status')"

	sleep "$(( ( RANDOM % ${{ env.RETRY }} ) + ${{ env.RETRY }} ))s"
	done

	# .. check its exit code
	expected_exit_code=0
	actual_exit_code="$(with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
	${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
	"balena ps -aq \| xargs balena inspect \
	\| jq -re '.[] \| select(.Name \| contains(\"sut_\")).State.ExitCode'")"

	[[ $expected_exit_code -eq $actual_exit_code ]] \|\| false

	env:
	ATTEMPTS: 2

	- name: provision Ubuntu ephemeral SUT
	id: ubuntu-sut
	if: \|
	github.event_name == 'pull_request' &&
	github.event.action != 'closed' &&
	matrix.target == 'compose-private-pki'
	run: \|
	set -ue

	[[ '${{ matrix.verbose }}' =~ on\|On\|Yes\|yes\|true\|True ]] && set -x

	function cleanup() {
	rm -f user-data.yml
	}
	trap 'cleanup' EXIT

	aws sts get-caller-identity

	# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/user-data.html
	# https://cloudinit.readthedocs.io/en/latest/reference/modules.html#update-etc-hosts
	cat << EOF > user-data.yml
	#cloud-config
	output : { all : '\| tee -a /var/log/cloud-init-output.log' }
	manage_etc_hosts: api.${{ matrix.subdomain }}.${{ matrix.dns_tld }}

	packages:
	- git
	- jq
	- wget

	write_files:
	- path: /root/.env
	permissions: 0600
	content: \|
	DNS_TLD=${{ matrix.subdomain }}.${{ matrix.dns_tld }}
	PRODUCTION_MODE=false
	VERBOSE=${{ matrix.verbose }}

	- path: /root/functions
	permissions: 0755
	content: \|
	# https://coderwall.com/p/--eiqg/exponential-backoff-in-bash
	function with_backoff() {
	local max_attempts=${ATTEMPTS-5}
	local timeout=${TIMEOUT-1}
	local attempt=0
	local exitCode=0

	set +e
	while [[ $attempt < $max_attempts ]]
	do
	"$@"
	exitCode=$?

	if [[ $exitCode == 0 ]]
	then
	break
	fi

	echo "Failure! Retrying in $timeout.." 1>&2
	sleep "$timeout"
	attempt=$(( attempt + 1 ))
	timeout=$(( timeout * 2 ))
	done

	if [[ $exitCode != 0 ]]
	then
	echo "You've failed me for the last time! ($*)" 1>&2
	fi
	set -e
	return $exitCode
	}

	# docs/getting-started.md
	- path: /root/getting-started.sh
	permissions: 0755
	content: \|
	#!/usr/bin/env bash

	set -ax

	[[ '${{ matrix.verbose }}' =~ on\|On\|Yes\|yes\|true\|True ]] && set -x

	source /root/functions

	apt-get update
	which openssl \|\| apt-get install -y make openssl
	which git \|\| apt-get install -y make git
	which jq \|\| apt-get install -y make jq
	which make \|\| apt-get install make

	which yq \|\| with_backoff wget -q https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq
	chmod +x /usr/bin/yq
	yq --version

	which docker \|\| curl -fsSL https://get.docker.com \| sh -
	usermod -aG docker ubuntu
	systemctl enable docker && systemctl start docker
	chown ubuntu:docker /var/run/docker.sock

	id -u balena \|\| useradd -s /bin/bash -m -G docker,sudo balena
	echo 'balena ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/balena

	while ! docker ps; do sleep $(((RANDOM%3)+1)); done
	with_backoff docker login \
	--username='${{ secrets.DOCKERHUB_USER }}' \
	--password='${{ secrets.DOCKERHUB_TOKEN }}'

	with_backoff docker login ghcr.io \
	--username=token \
	--password=${{ secrets.GITHUB_TOKEN }}

	if [ ! -f /sys/fs/cgroup/cgroup.controllers ]; then
	echo "cgroups v2 is disabled"
	else
	echo "cgroups v2 is enabled"
	source /etc/default/grub
	sed -i '/GRUB_CMDLINE_LINUX/d' /etc/default/grub
	echo GRUB_CMDLINE_LINUX=$(printf '\"%s systemd.unified_cgroup_hierarchy=0\"\n' "${GRUB_CMDLINE_LINUX}") > /etc/default/grub
	update-grub
	reboot
	fi

	# cloud-init runs as root
	# (e.g.) https://cloudinit.readthedocs.io/en/latest/reference/merging.html#example-cloud-config
	runcmd:
	- '/root/getting-started.sh' # FIXME: this may run before the script is written
	EOF

	for subnet_id in ${{ env.AWS_VPC_SUBNET_IDS }}; do
	# spot, on-demand
	for market_type in ${{ vars.MARKET_TYPES }}; do
	for instance_type in ${AWS_EC2_INSTANCE_TYPES}; do
	# https://docs.aws.amazon.com/cli/latest/reference/ec2/run-instances.html
	response="$(aws ec2 run-instances \
	--image-id "${{ matrix.ami }}" \
	--launch-template "LaunchTemplateId=${AWS_EC2_LAUNCH_TEMPLATE},Version=${AWS_EC2_LT_VERSION}" \
	--instance-type "${instance_type}" \
	$([[ $market_type =~ spot ]] && echo '--instance-market-options MarketType=spot') \
	--security-group-ids "${AWS_VPC_SECURITY_GROUP_IDS}" \
	--subnet-id "${subnet_id}" \
	--key-name '${{ steps.generate-key-pair.outputs.key_name }}' \
	--associate-public-ip-address \
	--user-data file://user-data.yml \
	--tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=open-balena-tests},{Key=matrix.target,Value=${{ matrix.target }}},{Key=MarketType,Value=${market_type}},{Key=Owner,Value=${{ env.AWS_IAM_USERNAME }}},{Key=GITHUB_SHA,Value=${GITHUB_SHA}-tests},{Key=GITHUB_RUN_ID,Value=${GITHUB_RUN_ID}-tests},{Key=GITHUB_RUN_NUMBER,Value=${GITHUB_RUN_NUMBER}-tests},{Key=GITHUB_RUN_ATTEMPT,Value=${GITHUB_RUN_ATTEMPT}-tests}]" \|\| true)"

	[[ -n $response ]] && break
	done
	[[ -n $response ]] && break
	done
	[[ -n $response ]] && break
	done

	[[ -z $response ]] && exit 1

	instance_id="$(echo "${response}" \| jq -r '.Instances[].InstanceId')"
	echo "instance_id=${instance_id}" >> $GITHUB_OUTPUT

	aws ec2 wait instance-running --instance-ids "${instance_id}"
	aws ec2 wait instance-status-ok --instance-ids "${instance_id}"

	env:
	AWS_DEFAULT_REGION: ${{ vars.AWS_REGION \|\| 'us-east-1' }}
	COMMIT: ${{ github.event.pull_request.head.sha \|\| github.event.head_commit.id \|\| github.event.pull_request.head.ref }}

	- name: SUT&DUT (compose)
	if: \|
	github.event_name == 'pull_request' &&
	github.event.action != 'closed' &&
	matrix.target == 'compose-private-pki'
	run: \|
	set -ue

	[[ '${{ matrix.verbose }}' =~ on\|On\|Yes\|yes\|true\|True ]] && set -x

	function log_output() {
	rm -f "{HOME}/.ssh/config"

	aws ssm list-command-invocations \
	--details \
	--output text \
	--command-id "${id}" \|\| true

	aws logs describe-log-streams \
	--log-group-name open-balena-tests \
	--log-stream-name-prefix "${id}" \|\| true

	aws logs put-retention-policy \
	--log-group-name open-balena-tests \
	--retention-in-days "${{ env.AWS_LOGS_RETENTION }}" \|\| true
	}
	trap 'log_output' EXIT

	# docs/getting-started.md
	CMDS="set -ax \
	&& cloud-init status --wait --long && cat </var/log/cloud-init-output.log \
	&& sudo -u balena git clone https://token:${{ steps.ephemeral.outputs.token }}/${{ github.repository }}.git /home/balena/open-balena \
	&& cd /home/balena/open-balena \
	&& sudo -u balena git checkout ${COMMIT}
	&& source /root/.env \
	&& sudo -Eu balena make up \
	&& sudo -u balena make self-signed \
	&& sudo -u balena make verify \
	&& sudo -u balena make restart \
	&& sudo -u balena docker compose logs -f sut




	# https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager-getting-started-enable-ssh-connections.html
	cat << EOF > "${HOME}/.ssh/config"
	host i-*
	StrictHostKeyChecking no
	UserKnownHostsFile /dev/null
	ProxyCommand sh -c "aws ssm start-session --target %h --document-name AWS-StartSSHSession --parameters 'portNumber=%p'"
	EOF

	# AWS-RunShellScript runs as root
	id="$(aws ssm send-command \
	--instance-ids ${{ steps.launch-ec2-instance.outputs.instance_id }} \
	--document-name AWS-RunShellScript \
	--comment "open-balena-tests@${{ matrix.target }}" \
	--parameters commands=["${CMDS}"] \
	--cloud-watch-output-config '{"CloudWatchLogGroupName":"open-balena-tests","CloudWatchOutputEnabled":true}' \| jq -r .Command.CommandId)"

	[[ -n $id ]] \|\| false

	while [[ $(aws logs describe-log-streams \
	--log-group-name open-balena-tests \
	--log-stream-name-prefix "${id}" \| jq -r '.logStreams\|length') -le 0 ]]; do
	echo '::info::waiting for logs...'
	done

	# https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines
	CYAN='\033[0;36m'; NC='\033[0m'; echo -e "::group::${CYAN}${{ env.SLUG }}${NC}"
	while [[ "$(aws ssm list-command-invocations --command-id "${id}" \
	\| jq -r '.CommandInvocations[].Status')" =~ InProgress ]]; do
	docker ps -q \| xargs docker logs --timestamps --follow --details \
	\|\| echo '::info::waiting for logs...'
	sleep $(((RANDOM%1) + 1))s
	done
	echo "::endgroup::"

	if ! [[ "$(aws ssm list-command-invocations --command-id "${id}" \
	\| jq -r '.CommandInvocations[].Status')" =~ Success ]]; then
	false
	fi

	env:
	AWS_DEFAULT_REGION: ${{ vars.AWS_REGION \|\| 'us-east-1' }}
	DOCKER_HOST: ssh://ec2-user@${{ steps.launch-ec2-instance.outputs.instance_id }}:22
	COMMIT: ${{ github.event.pull_request.head.sha \|\| github.event.head_commit.id \|\| github.event.pull_request.head.ref }}


























	- name: remove SSH key
	if: always() && matrix.target == 'balena-public-pki'
	continue-on-error: true
	run: \|
	set -ue

	[[ '${{ matrix.verbose \|\| 'false' }}' =~ on\|On\|Yes\|yes\|true\|True ]] && set -x

	source src/balena-tests/functions

	with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'

	with_backoff balena keys \| grep ${{ steps.provision-ssh-key.outputs.key_id }} \
	\| awk '{print $1}' \| xargs balena key rm --yes

	- name: destroy balena test device
	if: \|
	github.event_name == 'pull_request' &&
	github.event.action != 'closed' &&
	matrix.target == 'balena-public-pki'
	continue-on-error: true
	run: \|
	set -ue

	[[ '${{ matrix.verbose \|\| 'false' }}' =~ on\|On\|Yes\|yes\|true\|True ]] && set -x

	source src/balena-tests/functions

	with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'

	with_backoff balena device rm ${{ steps.register-test-device.outputs.balena_device_uuid }} --yes

	env:
	AWS_DEFAULT_REGION: ${{ vars.AWS_REGION \|\| 'us-east-1' }}

	# always destroy test EC2 instances even if the workflow is cancelled
	- name: destroy AWS test device
	if: always()
	run: \|
	set -ue

	[[ '${{ matrix.verbose \|\| 'false' }}' =~ on\|On\|Yes\|yes\|true\|True ]] && set -x

	source src/balena-tests/functions

	if [[ -n '${{ steps.balena-sut.outputs.instance_id }}' ]]; then
	with_backoff aws ec2 terminate-instances \
	--instance-ids ${{ steps.balena-sut.outputs.instance_id }}
	fi

	if [[ -n '${{ steps.ubuntu-sut.outputs.instance_id }}' ]]; then
	with_backoff aws ec2 terminate-instances \
	--instance-ids ${{ steps.ubuntu-sut.outputs.instance_id }}
	fi

	with_backoff aws ec2 describe-instances --filters Name=tag:GITHUB_SHA,Values=${GITHUB_SHA}-tests \
	\| jq -r .Reservations[].Instances[].InstanceId \
	\| xargs aws ec2 terminate-instances --instance-ids

	stale_instances=$(mktemp)
	aws ec2 describe-instances --filters \
	Name=tag:Name,Values=open-balena-tests \
	Name=instance-state-name,Values=running \
	\| jq -re '.Reservations[].Instances[].InstanceId + " " + .Reservations[].Instances[].LaunchTime' > ${stale_instances} \|\| true

	if test -s "${stale_instances}"; then
	while IFS= read -r line; do
	instance_id=$(echo ${line} \| awk '{print $1}')
	launch_time=$(echo ${line} \| awk '{print $2}')
	now=$(date +%s)
	then=$(date --date ${launch_time} +%s)
	days_since_launch=$(( (now - then) / 86400 ))
	if [[ -n $days_since_launch ]] && [[ $days_since_launch -ge 1 ]]; then
	with_backoff aws ec2 terminate-instances --instance-ids ${instance_id}
	fi
	done <${stale_instances}
	rm -f ${stale_instances}
	fi

	env:
	AWS_DEFAULT_REGION: ${{ vars.AWS_REGION \|\| 'us-east-1' }}

	# remove orphaned ACME DNS-01 validation records
	# https://letsencrypt.org/docs/challenge-types/#dns-01-challenge
	# FIXME: clean up older _acme-challenge.auto TXT records
	- name: cleanup-dns-records
	if: always() && matrix.target == 'balena-public-pki'
	continue-on-error: true
	run: \|
	set -ue

	[[ '${{ matrix.verbose \|\| 'false' }}' =~ on\|On\|Yes\|yes\|true\|True ]] && set -x

	if [[ -n '${{ steps.register-test-device.outputs.balena_device_uuid }}' ]]; then
	match="${{ steps.register-test-device.outputs.balena_device_uuid }}.${{ matrix.subdomain }}"

	zone_id="$(curl --silent --retry ${{ env.RETRY }} \
	"https://api.cloudflare.com/client/v4/zones?name=${{ matrix.dns_tld }}" \
	-H 'Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}' \| jq -r '.result[].id')"

	for record in "$(curl --silent --retry ${{ env.RETRY }} \
	"https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" \
	-H 'Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}' \
	\| jq -r --arg match "${match}" '.result[] \| select(((.type=="TXT") and (.name \| contains($match))))' \
	\| base64)"; do

	json="$(echo "${record}" \| base64 -d \| jq -r)"
	id="$(echo "${json}" \| jq -r .id)"
	name="$(echo "${json}" \| jq -r .name)"

	if [[ -n $id ]] && [[ -n $name ]]; then
	echo "::warning::Orphaned DNS record ${name} (${id})..."

	if [[ -z $DRY_RUN ]]; then
	curl -X DELETE --silent --retry ${{ env.RETRY }} \
	"https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records/${id}" \
	-H 'Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}'
	fi
	fi
	done
	fi

	env:
	DRY_RUN: false

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

wip #1

Workflow file

wip #1

Jobs

Run details

Workflow file for this run

GitHub Actions / .github/workflows/tests.yml