ci: sgx ci/cd

.github/scripts/gramine.sh

@@ -0,0 +1,43 @@
+#/bin/sh
+# this is to be ran in a docker container via an github action that has gramine set-up already e.g.,
+# notaryserverbuilds.azurecr.io/builder/gramine
+# with sgx hardware:
+# ./gramine.sh sgx
+#
+# without:
+# ./gramine.sh 
+##
+
+if [ -z "$1" ]
+  then
+     run='gramine-direct  notary-server &'
+
+   else
+      run='gramine-sgx  notary-server &'
+fi
+
+
+
+curl https://sh.rustup.rs -sSf | sh -s -- -y
+. "$HOME/.cargo/env"
+apt install libssl-dev
+
+gramine-sgx-gen-private-key
+SGX=1 make
+gramine-sgx-sign -m notary-server.manifest -o notary-server.sgx
+mr_enclave=$(gramine-sgx-sigstruct-view --verbose --output-format=json notary-server.sig |jq .mr_enclave)
+echo "mrenclave=$mr_enclave" >> "$GITHUB_OUTPUT"
+echo "#### sgx mrenclave" | tee >> $GITHUB_STEP_SUMMARY
+echo "\`\`\`${mr_enclave}\`\`\`" | tee >> $GITHUB_STEP_SUMMARY
+eval "$run"
+sleep 5
+
+if [ "$1" ]; then
+    curl 127.0.0.1:7047/info
+else
+    quote=$(curl 127.0.0.1:7047/info | jq .quote.rawQuote)
+    echo $quote
+    echo "quote=$quote" >> $GITHUB_OUTPUT
+    echo "#### 🔒 signed quote ${quote}" | tee >> $GITHUB_STEP_SUMMARY
+    echo "${quote}" | tee >> $GITHUB_STEP_SUMMARY
+fi

.github/workflows/ci.yml

@@ -138,4 +138,11 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: lcov.info
-          fail_ci_if_error: true
+          fail_ci_if_error: true
+  trigger-deployment:
+    # doing this here due to feedback @ https://github.com/tlsnotary/tlsn/pull/631#issuecomment-2415806267
+    needs: tests-integration
+    uses: ./.github/workflows/tee-cd.yml
+    with:
+      # what this is supposed to do -> $ref is the tag: e.g., v0.1.0-alpha.7; pass the $ref string to the cd script and update reverse proxy / deploy 
+      ref: ${{  github.ref_name }}	

.github/workflows/tee-cd.yml

@@ -0,0 +1,156 @@
+name: azure-tee-release
+
+permissions:
+  contents: read
+  id-token: write
+  attestations: write
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'git branch'
+        required: false
+        default: 'dev'
+        type: string
+
+#on:
+#  release:
+#    types: [published]
+#    branches:
+#      - 'releases/**'
+
+env:
+  GIT_COMMIT_HASH: ${{ github.event.pull_request.head.sha || github.sha }}
+  GIT_COMMIT_TIMESTAMP: ${{ github.event.repository.updated_at}}
+  REGISTRY: notaryserverbuilds.azurecr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  update-reverse-proxy:
+    permissions:
+      contents: write
+    environment: tee
+    runs-on: [self-hosted, linux]
+    outputs:
+      teeport: ${{ steps.portbump.outputs.newport}}
+      deploy: ${{ steps.portbump.outputs.deploy}}
+    steps:
+      - name: checkout repository
+        uses: actions/checkout@v4
+      - name: update caddyfile
+        id: portbump
+        env:
+          RELEASE_TAG: ${{ github.event.release.tag_name ||  inputs.ref }}
+        run: |
+          echo "tag: $RELEASE_TAG"
+          NEXT_PORT=$(bash cd-scripts/tee/azure/updateproxy.sh 'cd-scripts/tee/azure/Caddyfile' $RELEASE_TAG)
+          echo "newport=$NEXT_PORT" >> $GITHUB_OUTPUT
+          echo "new deploy port: $NEXT_PORT  🚀" >> $GITHUB_STEP_SUMMARY
+          chmod +r -R cd-scripts/tee/azure/
+      - name: Deploy updated Caddyfile to server
+        if: ${{ steps.portbump.outputs.deploy == 'new' }}
+        uses: appleboy/[email protected]
+        with:
+          host: ${{ secrets.AZURE_TEE_PROD_HOST }}
+          username: ${{ secrets.AZURE_PROD_TEE_USERNAME }}
+          key: ${{ secrets.AZURE_TEE_PROD_KEY }}
+          source: "cd-scripts/tee/azure/Caddyfile"
+          target: "~/"
+      - name: Reload Caddy on server
+        if: ${{ steps.portbump.outputs.deploy == 'new' }}
+        uses: appleboy/[email protected]
+        with:
+          host: ${{ secrets.AZURE_TEE_PROD_HOST }}
+          username: ${{ secrets.AZURE_PROD_TEE_USERNAME }}
+          key: ${{ secrets.AZURE_TEE_PROD_KEY }}
+          script: |
+            sudo cp ~/cd-scripts/tee/azure/Caddyfile /etc/caddy/Caddyfile
+            sudo systemctl reload caddy
+  build-measure:
+      environment: tee
+      runs-on: [self-hosted, linux]
+      needs: [ update-reverse-proxy ]
+      container:
+        image: notaryserverbuilds.azurecr.io/prod/gramine
+        credentials:
+          username: notaryserverbuilds
+          password: ${{ secrets.AZURE_CR_BUILDS_PW }}
+        env:
+          GIT_COMMIT_HASH: ${{ github.event.pull_request.head.sha || github.sha }}
+        volumes:
+         - /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket
+        options: "--device /dev/sgx_enclave"
+      steps:
+        - name: get code
+          uses: actions/checkout@v4
+        - name: sccache
+          if: github.event_name != 'release'
+          # && github.event_name != 'workflow_dispatch'
+          uses: mozilla-actions/[email protected]
+        - name: set rust env for scc
+          if: github.event_name != 'release'
+          # && github.event_name != 'workflow_dispatch'
+          run: |
+            echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
+            echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
+        - name: reverse proxy port
+          run: echo "${{needs.update-reverse-proxy.outputs.teeport}}" | tee >> $GITHUB_STEP_SUMMARY
+        - name: get hardware measurement
+          working-directory: ${{ github.workspace }}/crates/notary/server/tee
+          run: |
+               chmod +x  ../../../../.github/scripts/gramine.sh && ../../../../.github/scripts/gramine.sh sgx
+  artifact-deploy:
+      environment: tee
+      runs-on: [self-hosted, linux]
+      needs: [ build-measure, update-reverse-proxy ]
+      steps:
+        - name: auth to registry
+          uses: docker/login-action@v3
+          with:
+            registry: notaryserverbuilds.azurecr.io
+            username: notaryserverbuilds
+            password: ${{ secrets.AZURE_CR_BUILDS_PW }}
+        - name: get code
+          uses: actions/checkout@v4
+        - name: Set up Docker Buildx
+          uses: docker/setup-buildx-action@v3
+        - name: Get Git commit timestamps
+          run: echo "TIMESTAMP=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV
+        - name: Build and push
+          id: deploypush
+          uses: docker/build-push-action@v6
+          with:
+            provenance: mode=max
+            no-cache: true
+            context: ${{ github.workspace }}/crates/notary/server/tee
+            push: true
+            tags: notaryserverbuilds.azurecr.io/prod/notary-sgx:${{ env.GIT_COMMIT_HASH }}
+            labels: ${{needs.update-reverse-proxy.outputs.teeport}}
+          env:
+            # reproducible builds: https://github.com/moby/buildkit/blob/master/docs/build-repro.md#source_date_epoch
+            SOURCE_DATE_EPOCH: ${{ env.TIMESTAMP }}
+        - name: Generate SBOM
+          uses: anchore/sbom-action@v0
+          with:
+            image:  notaryserverbuilds.azurecr.io/prod/notary-sgx:${{ env.GIT_COMMIT_HASH }}
+            format: 'cyclonedx-json'
+            output-file: 'sbom.cyclonedx.json'
+          # attestation section ::
+          # https://docs.docker.com/build/ci/github-actions/attestations/
+        - name: Attest
+          uses: actions/attest-build-provenance@v1
+          with:
+            subject-name: notaryserverbuilds.azurecr.io/prod/notary-sgx
+            subject-digest: ${{ steps.deploypush.outputs.digest }}
+            push-to-registry: true
+        -    
+          name: run
+          run: |
+            if [[ ${{ needs.update-reverse-proxy.outputs.deploy }} == 'new' ]]; then
+              docker run --device /dev/sgx_enclave --device /dev/sgx_provision --volume=/var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket -p  ${{needs.update-reverse-proxy.outputs.teeport}}:7047 notaryserverbuilds.azurecr.io/prod/notary-sgx:${{ env.GIT_COMMIT_HASH }} &
+            else
+              old=$(docker ps --filter "name=${{needs.update-reverse-proxy.outputs.teeport}}")
+              docker rm -f $old
+              docker run --name ${{needs.update-reverse-proxy.outputs.teeport}} --device /dev/sgx_enclave --device /dev/sgx_provision --volume=/var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket -p  ${{needs.update-reverse-proxy.outputs.teeport}}:7047 notaryserverbuilds.azurecr.io/prod/notary-sgx:${{ env.GIT_COMMIT_HASH }} &
+            fi

.github/workflows/tee-ci.yml

@@ -0,0 +1,41 @@
+name: tee-build
+
+on:
+  push:
+    branches: [ "dev" ]
+  pull_request:
+    branches: [ "dev" ]
+
+concurrency:
+  group: ${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+   build-measure-emulated:
+      environment: tee
+      runs-on: [self-hosted, linux]
+      container:
+        image: notaryserverbuilds.azurecr.io/prod/gramine
+        credentials:
+          username: notaryserverbuilds
+          password: ${{ secrets.AZURE_CR_BUILDS_PW }}
+        env:
+          GIT_COMMIT_HASH: ${{ github.event.pull_request.head.sha || github.sha }}
+      steps:
+        - name: get code
+          uses: actions/checkout@v4
+        - name: sccache
+          if: github.event_name != 'release'
+          # && github.event_name != 'workflow_dispatch'
+          uses: mozilla-actions/[email protected]
+        - name: set rust env for scc
+          if: github.event_name != 'release'
+          # && github.event_name != 'workflow_dispatch'
+          run: |
+            echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
+            echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
+
+        - name: get emulated measurement (call gramine.sh without the sgx arg)
+          working-directory: ${{ github.workspace }}/crates/notary/server/tee
+          run: |
+             bash .github/scripts/gramine.sh

cd-scripts/tee/azure/Caddyfile

@@ -0,0 +1,90 @@
+#
+#  global block =>
+#    email is for acme
+# # # #
+
+{
+  key_type p256
+  email [email protected] # for acme
+  servers {
+    metrics
+  }
+  log {
+    output stdout
+    format console {
+      time_format common_log
+      time_local
+    }
+    level DEBUG
+  }
+}
+
+#
+#    server block, acme turned on (default when using dns)
+#    reverse proxy with fail_duration + lb will try upstreams sequentially (fallback)
+#    e.g. => `reverse_proxy :4000 :5000 10.10.10.10:1000 tlsnotary.org:443`
+#    will always deliver to :4000 if its up, but if :4000 is down for more than 4s it trys the next one
+# # # #
+
+notary.codes {
+    handle_path /v0.1.0-alpha.8* {
+        reverse_proxy :4002 :3333 {
+            lb_try_duration 4s
+            fail_duration 10s
+            lb_policy header X-Upstream {
+                fallback first
+            }
+        }
+    }
+    handle_path /v0.1.0-alpha.7* {
+        reverse_proxy :4002 :3333 {
+            lb_try_duration 4s
+            fail_duration 10s
+            lb_policy header X-Upstream {
+                fallback first
+            }
+        }
+    }
+    handle_path /v0.1.0-alpha.6* {
+        reverse_proxy :4001 :3333 {
+            lb_try_duration 4s
+            fail_duration 10s
+            lb_policy header X-Upstream {
+                fallback first
+            }
+        }
+    }
+
+    handle_path /nightly* {
+     reverse_proxy :3333 {
+     lb_try_duration 4s
+     fail_duration 10s
+     lb_policy header X-Upstream {
+       fallback first
+      }
+     }
+    }
+
+    handle_path /proxy* {
+     reverse_proxy :55688 proxy.notary.codes:443 {
+     lb_try_duration 4s
+     fail_duration 10s
+     lb_policy header X-Upstream {
+       fallback first
+      }
+     }
+    }
+
+    handle  {
+     root * /srv
+     file_server
+    }
+
+    handle_errors {
+        @404 {
+            expression {http.error.status_code} == 404
+        }
+        rewrite @404 /index.html
+        file_server
+    }
+}

cd-scripts/tee/azure/prometheus.yaml

@@ -0,0 +1,7 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: caddy
+    static_configs:
+      - targets: ['localhost:2019']