ci: sgx ci/cd

.github/scripts/gramine.sh

@@ -0,0 +1,44 @@
+#/bin/sh
+# this is to be ran in a docker container via an github action that has gramine set-up already e.g.,
+# notaryserverbuilds.azurecr.io/builder/gramine
+# with sgx hardware:
+# ./gramine.sh sgx
+#
+# without:
+# ./gramine.sh 
+##
+
+if [ -z "$1" ]
+  then
+     run='gramine-direct  notary-server &'
+
+   else
+      run='gramine-sgx  notary-server &'
+fi
+
+
+
+curl https://sh.rustup.rs -sSf | sh -s -- -y
+. "$HOME/.cargo/env"
+apt install libssl-dev
+
+gramine-sgx-gen-private-key
+SGX=1 make
+gramine-sgx-sign -m notary-server.manifest -o notary-server.sgx
+mr_enclave=$(gramine-sgx-sigstruct-view --verbose --output-format=json notary-server.sig |jq .mr_enclave)
+echo "mrenclave=$mr_enclave" >> "$GITHUB_OUTPUT"
+echo "#### sgx mrenclave" | tee >> $GITHUB_STEP_SUMMARY
+echo "\`\`\`${mr_enclave}\`\`\`" | tee >> $GITHUB_STEP_SUMMARY
+mkdir config && cp config.yaml config && mkdir /tee && cp -R ../tee/* /tee/ && cd /tee &&
+eval "$run"
+sleep 5
+
+if [ "$1" ]; then
+    curl 127.0.0.1:7047/info
+else
+    quote=$(curl 127.0.0.1:7047/info | jq .quote.rawQuote)
+    echo $quote
+    echo "quote=$quote" >> $GITHUB_OUTPUT
+    echo "#### 🔒 signed quote ${quote}" | tee >> $GITHUB_STEP_SUMMARY
+    echo "${quote}" | tee >> $GITHUB_STEP_SUMMARY
+fi

.github/workflows/tee-cd.yml

@@ -0,0 +1,176 @@
+name: azure-tee-release
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'git branch'
+        required: false
+        default: 'dev'
+        type: string
+
+#on:
+#  release:
+#    types: [published]
+#    branches:
+#      - 'releases/**'
+
+env:
+  GIT_COMMIT_HASH: ${{ github.event.pull_request.head.sha || github.sha }}
+  GIT_COMMIT_TIMESTAMP: ${{ github.event.repository.updated_at}}
+
+jobs:
+  update-reverse-proxy:
+    environment: tee
+    runs-on: [self-hosted, linux]
+    outputs:
+      teeport: ${{ steps.portbump.outputs.newport}}
+      deploy: ${{ steps.portbump.outputs.deploy}}
+    steps:
+      - name: checkout repository
+        uses: actions/checkout@v4
+      - name: update caddyfile
+        id: portbump
+        env:
+          RELEASE_TAG: ${{ github.event.release.tag_name ||  inputs.ref }}
+
+        run: |
+          NEXT_PORT=$(bash cd-scripts/tee/azure/updateproxy.sh 'cd-scripts/tee/azure/Caddyfile' $RELEASE_TAG)
+          echo "newport=$NEXT_PORT" >> $GITHUB_OUTPUT
+
+      - name: Deploy updated Caddyfile to server
+        if: ${{ steps.portbump.outputs.deploy == 'new' }}
+        uses: appleboy/[email protected]
+        with:
+          host: ${{ secrets.AZURE_TEE_PROD_HOST }}
+          username: ${{ secrets.AZURE_PROD_TEE_USERNAME }}
+          key: ${{ secrets.AZURE_TEE_PROD_KEY }}
+          source: "cd-scripts/tee/azure/Caddyfile"
+          target: "~/"
+
+      - name: Reload Caddy on server
+        if: ${{ steps.portbump.outputs.deploy == 'new' }}
+
+        uses: appleboy/[email protected]
+        with:
+          host: ${{ secrets.AZURE_TEE_PROD_HOST }}
+          username: ${{ secrets.AZURE_PROD_TEE_USERNAME }}
+          key: ${{ secrets.AZURE_TEE_PROD_KEY }}
+          script: |
+            sudo cp ~/cd-scripts/tee/azure/Caddyfile /etc/caddy/Caddyfile
+            sudo systemctl reload caddy
+  build-measure:
+      environment: tee
+      runs-on: [self-hosted, linux]
+      needs: [ update-reverse-proxy ]
+      container:
+        image: notaryserverbuilds.azurecr.io/prod/gramine
+        credentials:
+          username: notaryserverbuilds
+          password: ${{ secrets.AZURE_CR_BUILDS_PW }}
+        env:
+          GIT_COMMIT_HASH: ${{ github.event.pull_request.head.sha || github.sha }}
+        volumes:
+         - /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket
+        options: "--device /dev/sgx_enclave"
+      steps:
+        - name: get code
+          uses: actions/checkout@v4
+        - name: sccache
+          if: github.event_name != 'release'
+          # && github.event_name != 'workflow_dispatch'
+          uses: mozilla-actions/[email protected]
+        - name: set rust env for scc
+          if: github.event_name != 'release'
+          # && github.event_name != 'workflow_dispatch'
+          run: |
+            echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
+            echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
+        - name: reverse proxy port
+          run: echo "${{needs.update-reverse-proxy.outputs.teeport}}" | tee >> $GITHUB_STEP_SUMMARY
+
+        - name: get hardware measurement
+          working-directory: ${{ github.workspace }}/crates/notary/server/tee
+          run: |
+             bash .github/scripts/gramine.sh sgx
+
+  artifact-deploy:
+      if: ${{needs.update-reverse-proxy.outputs.deploy == 'new' }}
+      environment: tee
+      runs-on: [self-hosted, linux]
+      needs: [ build-measure, update-reverse-proxy ]
+      steps:
+        - name: auth to registry
+          uses: docker/login-action@v3
+          with:
+            registry: notaryserverbuilds.azurecr.io
+            username: notaryserverbuilds
+            password: ${{ secrets.AZURE_CR_BUILDS_PW }}
+        - name: get code
+          uses: actions/checkout@v4
+        - name: sccache
+          if: github.event_name != 'release'
+          # && github.event_name != 'workflow_dispatch'
+          uses: mozilla-actions/[email protected]
+        - name: set rust env for scc
+          if: github.event_name != 'release'
+          # && github.event_name != 'workflow_dispatch'
+          run: |
+            echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
+            echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
+        -
+          name: Set up Docker Buildx
+          uses: docker/setup-buildx-action@v3
+        - name: Build and push
+          uses: docker/build-push-action@v6
+          with:
+            context: ${{ github.workspace }}/crates/notary/server/tee
+            push: true
+            tags: notaryserverbuilds.azurecr.io/prod/notary-sgx:${{ env.GIT_COMMIT_HASH }}
+            labels: ${{needs.update-reverse-proxy.outputs.teeport}}
+        - name: run
+          run: |
+           docker run --device /dev/sgx_enclave --device /dev/sgx_provision --volume=/var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket -p  ${{needs.update-reverse-proxy.outputs.teeport}}:7047 notaryserverbuilds.azurecr.io/prod/notary-sgx:${{ env.GIT_COMMIT_HASH }} &
+
+  artifact-reload:
+      if: ${{needs.update-reverse-proxy.outputs.deploy != 'new' }}
+      environment: tee
+      runs-on: [self-hosted, linux]
+      needs: [ build-measure, update-reverse-proxy ]
+      steps:
+        - name: auth to registry
+          uses: docker/login-action@v3
+          with:
+            registry: notaryserverbuilds.azurecr.io
+            username: notaryserverbuilds
+            password: ${{ secrets.AZURE_CR_BUILDS_PW }}
+        - name: get code
+          uses: actions/checkout@v4
+        - name: sccache
+          if: github.event_name != 'release'
+          # && github.event_name != 'workflow_dispatch'
+          uses: mozilla-actions/[email protected]
+        - name: set rust env for scc
+          if: github.event_name != 'release'
+          # && github.event_name != 'workflow_dispatch'
+          run: |
+            echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
+            echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
+        -
+          name: Set up Docker Buildx
+          uses: docker/setup-buildx-action@v3
+        - name: Build and push
+          uses: docker/build-push-action@v6
+          with:
+            context: ${{ github.workspace }}/crates/notary/server/tee
+            push: true
+            tags: notaryserverbuilds.azurecr.io/prod/notary-sgx:${{ env.GIT_COMMIT_HASH }}
+            labels: ${{needs.update-reverse-proxy.outputs.teeport}}
+        - name: run
+          run: |
+            old=$($ docker ps --filter "name=${{needs.update-reverse-proxy.outputs.teeport}}")
+            docker rm -f $old
+            docker run --name ${{needs.update-reverse-proxy.outputs.teeport}} --device /dev/sgx_enclave --device /dev/sgx_provision --volume=/var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket -p  ${{needs.update-reverse-proxy.outputs.teeport}}:7047 notaryserverbuilds.azurecr.io/prod/notary-sgx:${{ env.GIT_COMMIT_HASH }} &
+
+
+

.github/workflows/tee-ci.yml

@@ -0,0 +1,41 @@
+name: tee-build
+
+on:
+  push:
+    branches: [ "dev" ]
+  pull_request:
+    branches: [ "dev" ]
+
+concurrency:
+  group: ${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+   build-measure-emulated:
+      environment: tee
+      runs-on: [self-hosted, linux]
+      container:
+        image: notaryserverbuilds.azurecr.io/prod/gramine
+        credentials:
+          username: notaryserverbuilds
+          password: ${{ secrets.AZURE_CR_BUILDS_PW }}
+        env:
+          GIT_COMMIT_HASH: ${{ github.event.pull_request.head.sha || github.sha }}
+      steps:
+        - name: get code
+          uses: actions/checkout@v4
+        - name: sccache
+          if: github.event_name != 'release'
+          # && github.event_name != 'workflow_dispatch'
+          uses: mozilla-actions/[email protected]
+        - name: set rust env for scc
+          if: github.event_name != 'release'
+          # && github.event_name != 'workflow_dispatch'
+          run: |
+            echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
+            echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
+
+        - name: get emulated measurement (call gramine.sh without the sgx arg)
+          working-directory: ${{ github.workspace }}/crates/notary/server/tee
+          run: |
+             bash .github/scripts/gramine.sh

cd-scripts/tee/azure/Caddyfile

@@ -0,0 +1,52 @@
+{
+  key_type p256
+  email [email protected] # for acme
+  servers {
+    metrics
+  }
+  log {
+    output stdout
+    format console {
+      time_format common_log
+      time_local
+    }
+    level DEBUG
+  }
+}
+
+notary.codes {
+
+    handle_path /v0.1.0-alpha.6* {
+     reverse_proxy :6061 :3333 tlsnotary.org:443 {
+     lb_try_duration 4s
+     fail_duration 10s
+     lb_policy header X-Upstream {
+       fallback first
+      }
+     }
+    }
+
+    handle_path /v0.1.0-alpha.7* {
+     reverse_proxy :6062 :3333 tlsnotary.org:443 {
+     lb_try_duration 4s
+     fail_duration 10s
+     lb_policy header X-Upstream {
+       fallback first
+      }
+     }
+    }
+
+
+    handle  {
+     root * /srv
+     file_server
+    }
+
+    handle_errors {
+        @404 {
+            expression {http.error.status_code} == 404
+        }
+        rewrite @404 /index.html
+        file_server
+    }
+}

cd-scripts/tee/azure/prometheus.yaml

@@ -0,0 +1,7 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: caddy
+    static_configs:
+      - targets: ['localhost:2019']

cd-scripts/tee/azure/updateproxy.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+# Variables (Update these as needed)x
+CADDYFILE=${1:-/etc/caddy/Caddyfile}  # Path to your Caddyfile
+GIT_COMMIT_HASH=${2:-dev}
+BASE_PORT=6061                      # The starting port for your reverse_proxy directives
+
+# Function to check if handle_path for the given commit hash exists
+handle_path_exists() {
+    local commit_hash=$1
+    grep -q "handle_path /${commit_hash}\*" "$CADDYFILE"
+}
+
+# Function to extract the port for a given commit hash
+extract_port_for_commit() {
+    local commit_hash=$1
+    grep -Pzo "handle_path /${commit_hash}\* \{\n\s*reverse_proxy :(.*) " "$CADDYFILE" | grep -Poa "reverse_proxy :(.*) " | awk '{print $2}'
+}
+
+# Function to get the last port in the Caddyfile
+get_last_port() {
+    grep -Po "reverse_proxy :([0-9]+)" "$CADDYFILE" | awk -F: '{print $2}' | sort -n | tail -1
+}
+
+# Function to add a new handle_path block with incremented port inside notary.codes block
+add_new_handle_path() {
+    local new_port=$1
+    local commit_hash=$2
+
+    # Use a temporary file for inserting the handle_path block
+    tmp_file=$(mktemp)
+
+    # Add the new handle_path in the notary.codes block
+    awk -v port="$new_port" -v hash="$commit_hash" '
+        /tee\.notary\.codes \{/ {
+            print;
+            print "    handle_path /" hash "* {";
+            print "        reverse_proxy :" port " :3333 tlsnotary.org:443 {";
+            print "            lb_try_duration 4s";
+            print "            fail_duration 10s";
+            print "            lb_policy header X-Upstream {";
+            print "                fallback first";
+            print "            }";
+            print "        }";
+            print "    }";
+            next;
+        }
+        { print }
+    ' "$CADDYFILE" > "$tmp_file"
+
+    # Overwrite the original Caddyfile with the updated content
+    mv "$tmp_file" "$CADDYFILE"
+
+}
+
+# Check if the commit hash already exists in a handle_path
+if handle_path_exists "$GIT_COMMIT_HASH"; then
+    existing_port=$(extract_port_for_commit "$GIT_COMMIT_HASH")
+    echo "${existing_port:1}"
+    exit 0
+else
+    # Get the last port used and increment it
+    last_port=$(get_last_port)
+    if [[ -z "$last_port" ]]; then
+        last_port=$BASE_PORT
+    fi
+    new_port=$((last_port + 1))
+
+    # Add the new handle_path block inside notary.codes block
+    add_new_handle_path "$new_port" "$GIT_COMMIT_HASH"
+    echo $new_port
+    exit 0
+fi