Merge branch 'scionproto-master' into upstream

.bazelignore

@@ -1,4 +1,6 @@
 bin
+doc/_build
+doc/venv
 docker/_build
-rules_openapi/tools/node_modules
+private/mgmtapi/tools/node_modules
 tools/lint/logctxcheck/testdata/src

.bazelrc

@@ -1,10 +1,12 @@
 ### common options for all subcommands (help, query, build, ...)
-common --show_timestamps
+common --show_timestamps --enable_platform_specific_config
+
 # connect to buchgr/bazel-remote cache
 # These flags can unfortunately not be specified for `common`, as they are not accepted by all subcommands (help, version, dump)
-build --remote_cache=grpc://localhost:9092 --experimental_remote_downloader=grpc://localhost:9092
-query --remote_cache=grpc://localhost:9092 --experimental_remote_downloader=grpc://localhost:9092
-fetch --remote_cache=grpc://localhost:9092 --experimental_remote_downloader=grpc://localhost:9092
+# The --experimental_remote_downloader_local_fallback is used as workaround for issue with rules_oci (https://github.com/bazel-contrib/rules_oci/issues/275).
+build --remote_cache=grpc://localhost:9092 --experimental_remote_downloader=grpc://localhost:9092 --experimental_remote_downloader_local_fallback=true
+query --remote_cache=grpc://localhost:9092 --experimental_remote_downloader=grpc://localhost:9092 --experimental_remote_downloader_local_fallback=true
+fetch --remote_cache=grpc://localhost:9092 --experimental_remote_downloader=grpc://localhost:9092 --experimental_remote_downloader_local_fallback=true
 
 ### options for build, test, run, clean, etc.
 # expose git version (etc) to bazel
@@ -13,6 +15,14 @@ build --workspace_status_command=./tools/bazel-build-env
 build --java_runtime_version=remotejdk_11
 # disable legacy_create_init for py_binary, py_test etc. This may eventually become the default.
 build --incompatible_default_to_explicit_init_py
+# Enable resolution of cc toolchain by go toolchain
+build --incompatible_enable_cc_toolchain_resolution
+build --flag_alias=file_name_version=//:file_name_version
+
+# include one of "--define gotags=sqlite_mattn" or "--define gotags=sqlite_modernc"
+# cannot be in common, because query chokes on it.
+build --define gotags=sqlite_modernc,netgo
+build:osx --define gotags=sqlite_modernc
 
 ### options for test
 test --build_tests_only --print_relative_test_log_paths --test_output=errors
@@ -25,7 +35,7 @@ test:unit_all --config=unit //...
 test:integration --test_tag_filters=integration,-lint
 test:integration_all --config=integration //...
 
-test:lint --test_tag_filters=lint --test_summary=terse --noshow_progress --experimental_convenience_symlinks=ignore //...
+test:lint --test_tag_filters=lint,write_src --test_summary=terse --noshow_progress --experimental_convenience_symlinks=ignore
 
 # run quietly, only display errors
 common:quiet --ui_event_filters=-warning,-info,-debug,-stdout,-stderr --noshow_progress

.bazelversion

@@ -1 +1 @@
-5.1.0
+6.4.0

.buildkite/cleanup-leftovers.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+echo "~~~ Cleaning up any leftovers"
+cntrs="$(docker ps -aq | grep -v -f <(docker ps -q --filter "name=go-module-proxy" --filter "name=bazel-remote-cache"))"
+[ -n "$cntrs" ] && { echo "Remove leftover containers..."; docker rm -f $cntrs; }
+echo "Remove leftover networks"
+docker network prune -f
+echo "Remove leftover volumes"
+docker volume prune -f
+
+rm -rf bazel-testlogs logs/* traces gen gen-cache /tmp/test-artifacts test-out.tar.gz

.buildkite/hooks/bazel-remote.yml

@@ -1,4 +1,5 @@
 version: "2.4"
+name: bazel_remote
 services:
   bazel-remote:
     container_name: bazel-remote-cache

.buildkite/hooks/go-module-proxy.yml

@@ -1,5 +1,6 @@
 ---
 version: "2.4"
+name: athens
 services:
   go-module-proxy:
     container_name: go-module-proxy

.buildkite/hooks/pre-command

@@ -11,16 +11,29 @@ printenv PATH
 # Install build tools on first job for this runner.
 .buildkite/provision-agent.sh
 
-# Clean up left-overs from previous run
-PRE_COMMAND_SETUP=true . .buildkite/hooks/pre-exit
-
 set -euo pipefail
 
-echo "--- Increase receive network buffer size"
+echo "~~~ Increase receive network buffer size"
 sudo sysctl -w net.core.rmem_max=1048576
 
-echo "--- Setting up bazel environment"
 
+# Export SCION_VERSION as environment variable, unless it's set from outside (e.g. as variable for this build)
+# Note that this precommand hook runs even in "step zero" of the pipeline, where we `buildkite-agent upload'.
+# With this, the SCION_VERSION can be interpolated by the agent throughout the pipeline yaml.
+if [ -z ${SCION_VERSION+x} ]; then
+    echo "~~~ Export SCION_VERSION"
+    if [ "$BUILDKITE_PIPELINE_SLUG" == "scion" ]; then
+        # Shorten the git version to omit commit information, improving cache reuse.
+        # The format of git-version is "<tag>-<number-of-commits-since-the-tag>-<commit-short-hash>"
+        # This will be shortened to "<tag>-modified-ci"
+        export SCION_VERSION=$(tools/git-version | sed 's/-.*/-modified-ci/')
+    else
+        export SCION_VERSION=$(tools/git-version)
+    fi
+    echo SCION_VERSION=${SCION_VERSION}
+fi
+
+echo "~~~ Setting up bazel environment"
 if [ -z ${BAZEL_REMOTE_S3_ACCESS_KEY_ID+x} ]; then
     echo "S3 env not set, not starting bazel remote proxy"
     exit 0
@@ -33,20 +46,17 @@ rm -f $HOME/.bazelrc
 # --nostamp is required for better caching (only on non-release jobs).
 if [ "$BUILDKITE_PIPELINE_SLUG" == "scion" ]; then
     echo "build --nostamp" > $HOME/.bazelrc
-    # Also set a fixed GIT_VERSION so that the workspace_status_command always
-    # returns the same value on CI to improve cache reuse.
-    export GIT_VERSION="ci-fixed"
 else
     echo "build --stamp" > $HOME/.bazelrc
 fi
 echo "test --test_env CI" >> $HOME/.bazelrc
 
-echo "--- Starting bazel remote cache proxy"
+echo "~~~ Starting bazel remote cache proxy"
 
 # Start bazel remote cache proxy for S3
 # Note that S3 keys are injected by buildkite, see
 # https://buildkite.com/docs/pipelines/secrets#storing-secrets-with-the-elastic-ci-stack-for-aws
-docker-compose -f .buildkite/hooks/bazel-remote.yml -p bazel_remote up -d
+docker compose -f .buildkite/hooks/bazel-remote.yml up -d
 
-echo "--- Starting go module proxy"
-docker-compose -f .buildkite/hooks/go-module-proxy.yml -p athens up -d
+echo "~~~ Starting go module proxy"
+docker compose -f .buildkite/hooks/go-module-proxy.yml up -d

.buildkite/hooks/pre-exit

@@ -1,8 +1,7 @@
 #!/bin/bash
 
-
-if [ -f ".buildkite/hooks/bazel-remote.yml" -a -z "$PRE_COMMAND_SETUP" ]; then
-    echo "--- Uploading bazel-remote and go-module-proxy logs/metrics"
+if [ -f ".buildkite/hooks/bazel-remote.yml" ]; then
+    echo "~~~ Uploading bazel-remote and go-module-proxy logs/metrics"
     curl http://localhost:8080/metrics > bazel-remote-cache.metrics
     docker logs bazel-remote-cache &> bazel-remote-cache.log
 
@@ -11,19 +10,3 @@ if [ -f ".buildkite/hooks/bazel-remote.yml" -a -z "$PRE_COMMAND_SETUP" ]; then
 
     buildkite-agent artifact upload "bazel-remote-cache.*;go-module-proxy.*"
 fi
-
-echo "--- Cleaning up the topology"
-
-./scion.sh topo_clean
-
-echo "--- Cleaning up docker containers/networks/volumes"
-cntrs="$(docker ps -aq | grep -v -f <(docker ps -q --filter "name=go-module-proxy" --filter "name=bazel-remote-cache"))"
-[ -n "$cntrs" ] && { echo "Remove leftover containers..."; docker rm -f $cntrs; }
-
-echo "Remove leftover networks"
-docker network prune -f
-echo "Remove leftover volumes"
-docker volume prune -f
-
-echo "--- Cleaning up logs and artifacts"
-rm -rf bazel-testlogs logs/* traces gen gen-cache /tmp/test-artifacts

.buildkite/pipeline.yml

@@ -3,21 +3,72 @@ env:
 steps:
   - label: "Build :bazel:"
     command:
-      - bazel build --verbose_failures --announce_rc //:all
-      - bazel run --verbose_failures //docker:prod //docker:test
+      - bazel build --verbose_failures --announce_rc //:scion //:scion-ci
+      - bazel build --verbose_failures //docker:prod //docker:test
     key: build
+    artifact_paths:
+      - "scion_${SCION_VERSION}_amd64_linux.tar.gz"     # Note: SCION_VERSION interpolated by buildkite agent uploading pipeline, see pre-command hook
+      - "scion-ci_${SCION_VERSION}_amd64_linux.tar.gz"
+    plugins:
+      - scionproto/metahook#v0.3.0:
+          pre-artifact: |
+            gzip --to-stdout bazel-bin/scion.tar > scion_${SCION_VERSION}_amd64_linux.tar.gz
+            gzip --to-stdout bazel-bin/scion-ci.tar > scion-ci_${SCION_VERSION}_amd64_linux.tar.gz
+          post-artifact: |
+            cat << EOF | buildkite-agent annotate --style "info" --context "binaries"
+            #### Build outputs
+            - <a href="artifact://scion_${SCION_VERSION}_amd64_linux.tar.gz">SCION binaries</a>
+            - <a href="artifact://scion-ci_${SCION_VERSION}_amd64_linux.tar.gz">SCION test tools and utilities</a>
+            EOF
     retry: &automatic-retry
       automatic:
         - exit_status: -1 # Agent was lost
         - exit_status: 255 # Forced agent shutdown
     timeout_in_minutes: 10
   - wait
+  - label: "Package :debian: :openwrt: :rpm:"
+    command: |
+      make dist-deb BFLAGS="--file_name_version=${SCION_VERSION}"
+      make dist-openwrt BFLAGS="--file_name_version=${SCION_VERSION}"
+      make dist-rpm BFLAGS="--file_name_version=${SCION_VERSION}"
+    artifact_paths:
+      - "installables/scion_*.tar.gz"
+    plugins:
+      - scionproto/metahook#v0.3.0:
+          pre-artifact: |
+            set -x
+            pushd installables
+            tar -chaf scion_${SCION_VERSION}_deb_amd64.tar.gz *_${SCION_VERSION}_amd64.deb
+            tar -chaf scion_${SCION_VERSION}_deb_arm64.tar.gz *_${SCION_VERSION}_arm64.deb
+            tar -chaf scion_${SCION_VERSION}_deb_i386.tar.gz *_${SCION_VERSION}_i386.deb
+            tar -chaf scion_${SCION_VERSION}_deb_armel.tar.gz *_${SCION_VERSION}_armel.deb
+            tar -chaf scion_${SCION_VERSION}_openwrt_x86_64.tar.gz *_${SCION_VERSION}_x86_64.ipk
+            tar -chaf scion_${SCION_VERSION}_rpm_x86_64.tar.gz *_${SCION_VERSION}_x86_64.rpm
+            popd
+            ls installables
+          post-artifact: |
+            cat << EOF | buildkite-agent annotate --style "info" --context "packages"
+            #### Packages :debian:
+            - <a href="artifact://installables/scion_${SCION_VERSION}_deb_amd64.tar.gz">amd64</a>
+            - <a href="artifact://installables/scion_${SCION_VERSION}_deb_arm64.tar.gz">arm64</a>
+            - <a href="artifact://installables/scion_${SCION_VERSION}_deb_i386.tar.gz">i386</a>
+            - <a href="artifact://installables/scion_${SCION_VERSION}_deb_armel.tar.gz">armel</a>
+            #### Packages :openwrt:
+            - <a href="artifact://installables/scion_${SCION_VERSION}_openwrt_x86_64.tar.gz">x86_64</a>
+            #### Packages :rpm:
+            - <a href="artifact://installables/scion_${SCION_VERSION}_rpm_x86_64.tar.gz">x86_64</a>
+            EOF
+    key: dist
+    retry: *automatic-retry
   - label: "Unit Tests :bazel:"
     command:
       - bazel test --config=race --config=unit_all
     key: unit_tests
+    plugins:
+      - scionproto/metahook#v0.3.0:
+          pre-artifact: tar -chaf bazel-testlogs.tar.gz bazel-testlogs
     artifact_paths:
-      - "artifacts.out/**/*"
+      - bazel-testlogs.tar.gz
     retry: *automatic-retry
     timeout_in_minutes: 20
   - label: "Lint :bash:"
@@ -31,8 +82,8 @@ steps:
       - echo "--- go_deps.bzl"
       - mkdir -p /tmp/test-artifacts
       - cp go.mod go.sum go_deps.bzl /tmp/test-artifacts/
+      - make go.mod
       - make go_deps.bzl -B
-      - $(bazel info output_base 2>/dev/null)/external/go_sdk/bin/go mod tidy
       - diff -u /tmp/test-artifacts/go.mod go.mod
       - diff -u /tmp/test-artifacts/go.sum go.sum
       - diff -u /tmp/test-artifacts/go_deps.bzl go_deps.bzl
@@ -56,6 +107,7 @@ steps:
     timeout_in_minutes: 20
     key: check_generated
     retry: *automatic-retry
+  - wait
   - group: "End to End"
     key: e2e
     steps:
@@ -69,14 +121,20 @@ steps:
         - tools/await-connectivity
         - ./bin/scion_integration || ( echo "^^^ +++" && false )
         - ./bin/end2end_integration || ( echo "^^^ +++" && false )
-      plugins: &shutdown-scion-post-command
+      plugins: &scion-run-hooks
         - scionproto/metahook#v0.3.0:
+            pre-command: .buildkite/cleanup-leftovers.sh
             post-command: |
-              echo "--- Shutting down SCION topology"
+              echo "~~~ Shutting down SCION topology"
               ./scion.sh stop
-              echo "SCION topology successfully shut down"
-      artifact_paths:
-        - "artifacts.out/**/*"
+            pre-artifact: |
+              if [ -f "gen/scion-dc.yml" ]; then
+                  tools/dc collect_logs scion logs/
+              fi
+              tar -chaf test-out.tar.gz $(ls -d logs traces gen gen-cache) # ls -d to filter missing directories
+            pre-exit: .buildkite/cleanup-leftovers.sh
+      artifact_paths: &scion-run-artifact-paths
+        - test-out.tar.gz
       timeout_in_minutes: 15
       key: e2e_integration_tests_v2
       retry: *automatic-retry
@@ -90,25 +148,23 @@ steps:
         - tools/await-connectivity
         - ./bin/end2end_integration || ( echo "^^^ +++" && false )
         - ./tools/integration/revocation_test.sh
-      plugins: *shutdown-scion-post-command
-      artifact_paths:
-        - "artifacts.out/**/*"
+      plugins: *scion-run-hooks
+      artifact_paths: *scion-run-artifact-paths
       timeout_in_minutes: 15
       key: e2e_revocation_test_v2
       retry: *automatic-retry
     - label: "E2E: default :docker: (ping)"
       command:
         - echo "--- build"
-        - make build docker-images
+        - make build-dev docker-images
         - echo "--- start topology"
         - ./scion.sh topology -d
         - ./scion.sh run
         - tools/await-connectivity
         - echo "--- run tests"
         - ./bin/end2end_integration -d || ( echo "^^^ +++" && false )
-      plugins: *shutdown-scion-post-command
-      artifact_paths:
-        - "artifacts.out/**/*"
+      plugins: *scion-run-hooks
+      artifact_paths: *scion-run-artifact-paths
       timeout_in_minutes: 15
       key: docker_integration_e2e_default
       retry: *automatic-retry