EdenGCP #4
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: EdenGCP | |
| on: # yamllint disable-line rule:truthy | |
| workflow_run: | |
| workflows: | |
| - Publish | |
| types: | |
| - completed | |
| workflow_dispatch: | |
| inputs: | |
| test_type: | |
| description: 'test type' | |
| required: true | |
| type: choice | |
| options: | |
| - PR | |
| - docker | |
| pr_number: | |
| description: '[PR] number' | |
| required: false | |
| type: string | |
| eve_docker_tag: | |
| description: '[docker] eve tag' | |
| required: false | |
| eve_docker_reg: | |
| description: '[docker] eve registry' | |
| required: false | |
| type: choice | |
| options: | |
| - lfedge/eve | |
| - evebuild/danger | |
| - custom | |
| eve_docker_reg_custom: | |
| description: '[docker] eve custom registry' | |
| required: false | |
| type: string | |
| concurrency: | |
| group: ${{ github.workflow }} | |
| # yamllint disable rule:line-length | |
| jobs: | |
| integration: | |
| name: Integration test (${{ matrix.backend }};${{ matrix.hv }};${{ matrix.fs }}) | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| backend: ["gcp", "rol"] | |
| hv: ["kvm", "xen"] | |
| fs: ["zfs", "ext4"] | |
| steps: | |
| - name: Check | |
| run: | | |
| for addr in $(ip addr list|sed -En -e 's/.*inet ([0-9.]+).*/\1/p') | |
| do | |
| if echo "$addr" | grep -q -E "10.11.(12|13).[0-9]+"; then | |
| echo "$addr overlaps with test"; exit 1 | |
| fi | |
| if echo "$addr" | grep -q -E "10.8.0.[0-9]+"; then | |
| echo "$addr overlaps with vpn"; exit 1 | |
| fi | |
| done | |
| sudo df -h | |
| sudo swapoff -a | |
| sudo free | |
| - name: Check RoL secrets | |
| if: matrix.backend == 'rol' | |
| run: | | |
| if [ -z "$ROL_API_URL" ]; then echo "::error::ROL_API_URL is empty" && exit 1; fi | |
| if [ -z "$ROL_API_KEY" ]; then echo "::error::ROL_API_KEY is empty" && exit 1; fi | |
| if [ -z "$ROL_PROJECT" ]; then echo "::error::ROL_PROJECT is empty" && exit 1; fi | |
| if [ -z "$ROL_OVPN_CONF_BASE64" ]; then echo "::error::ROL_OVPN_CONF_BASE64 is empty" && exit 1; fi | |
| env: | |
| ROL_API_URL: ${{ secrets.ROL_API_URL }} | |
| ROL_API_KEY: ${{ secrets.ROL_API_KEY }} | |
| ROL_PROJECT: ${{ secrets.ROL_PROJECT }} | |
| ROL_OVPN_CONF_BASE64: ${{ secrets.ROL_OVPN_CONF_BASE64 }} | |
| - name: Setup packages | |
| run: | | |
| sudo apt update | |
| sudo apt install -y qemu binfmt-support qemu-user-static qemu-system-x86 qemu-system-aarch64 qemu-utils openvpn curl jq | |
| - id: 'gcpauth' | |
| if: matrix.backend == 'gcp' | |
| name: GCP Auth to Google Cloud SDK | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| project_id: ${{ secrets.GCP_PROJECT_ID }} | |
| credentials_json: ${{ secrets.GCP_SA_KEY }} | |
| create_credentials_file: true | |
| - name: GCP Set up Google Cloud SDK | |
| if: matrix.backend == 'gcp' | |
| uses: google-github-actions/setup-gcloud@v1 | |
| with: | |
| project_id: ${{ secrets.GCP_PROJECT_ID }} | |
| - name: GCP Clean | |
| if: matrix.backend == 'gcp' | |
| run: | | |
| gcloud compute instances delete eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} -q --zone=us-west1-a || echo "Does not exist" | |
| gcloud compute images delete eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} -q || echo "Does not exist" | |
| - name: Connect VPN | |
| id: connect_vpn | |
| timeout-minutes: 1 | |
| run: | | |
| echo "$OVPN_GCP_FILE" | base64 -d > ./config_gcp.ovpn | |
| echo "$OVPN_ROL_FILE" | base64 -d > ./config_rol.ovpn | |
| sudo openvpn --config ./config_${{ matrix.backend }}.ovpn --daemon | |
| until ip -f inet addr show tun0; do sleep 5; ip a; done | |
| echo "tunnel_ip=$(ip -f inet addr show tun0 | sed -En -e 's/.*inet ([0-9.]+).*/\1/p')" >> $GITHUB_OUTPUT | |
| env: | |
| OVPN_GCP_FILE: ${{ secrets.OVPN_FILE }} | |
| OVPN_ROL_FILE: ${{ secrets.ROL_OVPN_CONF_BASE64 }} | |
| - name: Check workflow dispatch inputs | |
| if: github.event_name == 'workflow_dispatch' | |
| run: | | |
| if [ "${{ github.event.inputs.test_type }}" == "docker" ]; then | |
| if [ -z "${{ github.event.inputs.eve_docker_tag }}" ]; then echo "::error::eve_docker_tag is empty" && exit 1; fi | |
| if [ -z "${{ github.event.inputs.eve_docker_reg }}" ]; then echo "::error::eve_docker_reg is empty" && exit 1; fi | |
| if [ "${{ github.event.inputs.eve_docker_reg }}" == "custom"]; then | |
| if [ -z "${{ github.event.inputs.eve_docker_reg_custom }}" ]; then echo "::error::eve_docker_reg_custom is empty" && exit 1; fi | |
| fi | |
| fi | |
| if [ "${{ github.event.inputs.test_type }}" == "PR" ]; then | |
| if [ -z "${{ github.event.inputs.pr_number }}" ]; then echo "::error::pr_number is empty" && exit 1; fi | |
| fi | |
| # We run this workflow for tags, release and default branches, | |
| # so we should pull required branch here | |
| # in case of workflow_dispatch ref will be empty (will point onto main branch) | |
| - name: Checkout EVE | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| path: eve | |
| ref: '${{ github.event.workflow_run.head_branch }}' | |
| - name: Force fetch annotated tags (workaround) | |
| working-directory: ./eve | |
| # Workaround for https://github.com/actions/checkout/issues/290 | |
| run: | | |
| git fetch --force --tags | |
| # we want to test the same commit as we publish during workflow_run | |
| - name: Prepare EVE (workflow) | |
| if: github.event_name == 'workflow_run' | |
| working-directory: ./eve | |
| run: | | |
| git reset --hard '${{ github.event.workflow_run.head_sha }}' | |
| git clean -f -x -d | |
| echo "EVE_TAG=$(make version)" >> "$GITHUB_ENV" | |
| echo "EVE_REGISTRY=lfedge/eve" >> "$GITHUB_ENV" | |
| - name: Prepare EVE (manual run) | |
| if: github.event_name == 'workflow_dispatch' | |
| working-directory: ./eve | |
| run: | | |
| if [ "${{ github.event.inputs.test_type }}" == "PR" ]; then | |
| git fetch origin pull/${{ github.event.inputs.pr_number }}/head && git checkout FETCH_HEAD | |
| echo "EVE_TAG=pr${{ github.event.inputs.pr_number }}" >> "$GITHUB_ENV" | |
| echo "EVE_REGISTRY=evebuild/danger" >> "$GITHUB_ENV" | |
| else | |
| echo "EVE_TAG=${{ github.event.inputs.eve_docker_tag }}" >> "$GITHUB_ENV" | |
| if [ "${{ github.event.inputs.eve_docker_reg }}" == "custom" ]; then | |
| echo "EVE_REGISTRY=${{ github.event.inputs.eve_docker_reg_custom }}" >> "$GITHUB_ENV" | |
| else | |
| echo "EVE_REGISTRY=${{ github.event.inputs.eve_docker_reg }}" >> "$GITHUB_ENV" | |
| fi | |
| fi | |
| # Use stored or the latest version of EDEN with subset of tests from EVE-OS repo | |
| - name: Prepare eden | |
| run: | | |
| if [ -f ${{ github.workspace }}/eve/tests/eden/eden-version ]; then | |
| EDEN_VERSION=$(cat ${{ github.workspace }}/eve/tests/eden/eden-version) | |
| else | |
| EDEN_VERSION=lfedge/eden:0.8.0 | |
| fi | |
| docker run -v $PWD:/out $EDEN_VERSION cp -a /eden/. /out/ | |
| sudo chown -R $(whoami) . | |
| - name: Eden setup | |
| run: | | |
| if [ "${{ matrix.backend }}" == "rol" ]; then | |
| devmodel='general' | |
| arch='arm64' | |
| netboot='true' | |
| tpm='false' | |
| else | |
| devmodel='GCP' | |
| arch='amd64' | |
| netboot='false' | |
| tpm='true' | |
| fi | |
| ./eden config add default --devmodel="${devmodel}" --arch="${arch}" | |
| ./eden config set default --key eve.registry --value="${{ env.EVE_REGISTRY }}" | |
| ./eden config set default --key eve.tag --value="${{ env.EVE_TAG }}" | |
| ./eden config set default --key eve.hv --value ${{ matrix.hv }} | |
| ./eden config set default --key eve.tpm --value "${tpm}" | |
| ./eden config set default --key adam.eve-ip --value ${{ steps.connect_vpn.outputs.tunnel_ip }} | |
| ./eden config set default --key registry.ip --value ${{ steps.connect_vpn.outputs.tunnel_ip }} | |
| ./eden config set default --key=eden.tests --value=${{ github.workspace }}/eve/tests/eden | |
| if [ "${{ matrix.fs }}" == "zfs" ]; then | |
| if [ "${{ matrix.backend }}" == "gcp" ]; then | |
| ./eden config set default --key=eve.disks --value=4 | |
| fi | |
| grub_options='set_global dom0_extra_args "$dom0_extra_args eve_install_zfs_with_raid_level "' | |
| else | |
| grub_options='' | |
| fi | |
| ./eden setup -v debug --grub-options="${grub_options}" --netboot="${netboot}" | |
| ./eden start | |
| - name: GCP Create VM | |
| if: matrix.backend == 'gcp' | |
| run: | | |
| export ipv4=`curl https://api.ipify.org/?format=json | jq ".ip" -r` | |
| ./eden utils gcp firewall --source-range $ipv4/32 --name eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} -k "${{ steps.gcpauth.outputs.credentials_file_path }}" | |
| ./eden utils gcp image --image-name eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} -k "${{ steps.gcpauth.outputs.credentials_file_path }}" upload | |
| ./eden utils gcp vm --image-name eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} --vm-name eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} -k "${{ steps.gcpauth.outputs.credentials_file_path }}" run | |
| sleep 100 | |
| BWD=$(./eden utils gcp vm get-ip --vm-name eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} -k "${{ steps.gcpauth.outputs.credentials_file_path }}") || { echo "no IP, probably VM does not exist"; exit 1; } | |
| echo "the IP is $BWD" | |
| ./eden utils gcp firewall -k "${{ steps.gcpauth.outputs.credentials_file_path }}" --source-range $BWD --name edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} || { echo "cannot set firewall"; exit 1; } | |
| - name: RoL rent RPi4 | |
| id: rol | |
| if: matrix.backend == 'rol' | |
| timeout-minutes: 60 | |
| run: | | |
| rent_id=$(./eden rol rent create -p "$ROL_PROJECT" -m raspberry --model pi_4_model_b_8gb -n GHAction-${{ github.run_number }}-snapshot-${{ matrix.hv }}-${{ matrix.fs }}) | |
| echo "id=$rent_id" >> $GITHUB_OUTPUT | |
| until [[ "$(./eden rol rent get -p "$ROL_PROJECT" -i $rent_id | jq -r .machineState)" == "Ready" ]]; do sleep 10; echo "Waiting for EVE to load"; done | |
| env: | |
| ROL_API_URL: ${{ secrets.ROL_API_URL }} | |
| ROL_API_KEY: ${{ secrets.ROL_API_KEY }} | |
| ROL_PROJECT: ${{ secrets.ROL_PROJECT }} | |
| - name: Eden EVE onboard | |
| run: | | |
| ./eden eve onboard | |
| - name: Test | |
| run: | | |
| if [ "${{ matrix.backend }}" == "gcp" ]; then export EDEN_TEST=gcp; fi | |
| EDEN_TEST_STOP=n ./eden test ${{ github.workspace }}/eve/tests/eden/workflow -v debug | |
| - name: Collect logs | |
| if: ${{ always() }} | |
| run: | | |
| ./eden log --format json > trace.log || echo "no log" | |
| ./eden info --format json > info.log || echo "no info" | |
| ./eden metric --format json > metric.log || echo "no metric" | |
| ./eden netstat --format json > netstat.log || echo "no netstat" | |
| docker logs eden_adam > adam.log 2>&1 || echo "no adam log" | |
| if [ "${{ matrix.backend }}" == "gcp" ]; then | |
| ./eden utils gcp vm log --vm-name eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} -k "${{ steps.gcpauth.outputs.credentials_file_path }}" > console.log || echo "no device log" | |
| else | |
| ./eden rol rent console-output -p "$ROL_PROJECT" -i "${{ steps.rol.outputs.id }}" > console.log || echo "no device log" | |
| fi | |
| env: | |
| ROL_API_URL: ${{ secrets.ROL_API_URL }} | |
| ROL_API_KEY: ${{ secrets.ROL_API_KEY }} | |
| ROL_PROJECT: ${{ secrets.ROL_PROJECT }} | |
| - name: Log counting | |
| if: ${{ always() }} | |
| run: | | |
| echo "::group::Total errors" | |
| echo "$(jq '.severity' trace.log|grep err|wc -l)" | |
| echo "::endgroup::" | |
| echo "::group::Errors by source" | |
| echo "errors by source: $(jq -s 'map(select(.severity|contains("err")))|group_by(.source)|map({"source": .[0].source, "total":length})|sort_by(.total)|reverse[]' trace.log)" | |
| echo "::endgroup::" | |
| echo "::group::Error log content duplicates" | |
| echo "$(jq -s 'map(select(.severity | contains("err")))|group_by(.content)|map(select(length>1))' trace.log)" | |
| echo "::endgroup::" | |
| echo "::group::Error log function filename duplicates" | |
| echo "$(jq -s 'map(select(.severity | contains("err")))|group_by(.filename)|map(select(length>10))|map({"source": .[0].source, "filename": .[0].filename, "function": .[0].function, "content": [.[].content], "total":length})|sort_by(.total)| reverse[]' trace.log)" | |
| echo "::endgroup::" | |
| echo "::group::Segfaults" | |
| echo "$(jq -s 'map(select(.content | contains("segfault at")))' trace.log)"|tee segfaults.log | |
| [ "$(jq length segfaults.log)" -gt 0 ] && echo "::warning::segfaults found, you can see them in Log counting->Segfaults section" | |
| echo "::endgroup::" | |
| - name: GCP Clean | |
| if: ${{ always() }} | |
| run: | | |
| gcloud compute firewall-rules delete eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} || echo "Does not exist" | |
| gcloud compute firewall-rules delete edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} || echo "Does not exist" | |
| gcloud compute instances delete eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} -q --zone=us-west1-a || echo "Does not exist" | |
| gcloud compute images delete eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} -q || echo "Does not exist" | |
| gsutil -o "Credentials:gs_service_key_file=${{ steps.gcpauth.outputs.credentials_file_path }}" rm gs://eve-live/eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}}.img.tar.gz || echo "Does not exists" | |
| - name: RoL Clean | |
| if: ${{ always() }} | |
| run: | | |
| if [ -z "${{ steps.rol.outputs.id }}" ]; then exit 0; fi | |
| ./eden rol rent close -p $ROL_PROJECT -i ${{ steps.rol.outputs.id }} | |
| env: | |
| ROL_API_URL: ${{ secrets.ROL_API_URL }} | |
| ROL_API_KEY: ${{ secrets.ROL_API_KEY }} | |
| ROL_PROJECT: ${{ secrets.ROL_PROJECT }} | |
| - name: Store raw test results | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: eden-report-${{ matrix.backend }}-${{ matrix.hv }}-${{ matrix.fs }} | |
| path: | | |
| ${{ github.workspace }}/trace.log | |
| ${{ github.workspace }}/info.log | |
| ${{ github.workspace }}/adam.log | |
| ${{ github.workspace }}/netstat.log | |
| ${{ github.workspace }}/metric.log | |
| ${{ github.workspace }}/console.log |