initial commit of s3 caching

felippeb · Mar 15, 2024 · 10be113 · 10be113
1 parent 29a2fdb
commit 10be113
Show file tree

Hide file tree

Showing 13 changed files with 66 additions and 19 deletions.
diff --git a/.github/actions/build-onedir-deps/action.yml b/.github/actions/build-onedir-deps/action.yml
@@ -24,22 +24,19 @@ inputs:
     type: string
     description: Seed used to invalidate caches
 
-
 env:
   COLUMNS: 190
   PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/
   PIP_EXTRA_INDEX_URL: https://pypi.org/simple
   RELENV_BUILDENV: 1
 
-
 runs:
   using: composite
 
   steps:
-
     - name: Cache Deps Onedir Package Directory
       id: onedir-pkg-cache
-      uses: actions/cache@v3.3.1
+      uses: ./.github/actions/cache
       with:
         path: artifacts/${{ inputs.package-name }}
         key: >

diff --git a/.github/actions/build-onedir-salt/action.yml b/.github/actions/build-onedir-salt/action.yml
@@ -43,7 +43,7 @@ runs:
 
     - name: Download Cached Deps Onedir Package Directory
       id: onedir-bare-cache
-      uses: actions/cache@v3.3.1
+      uses: ./.github/actions/cache
       with:
         path: artifacts/${{ inputs.package-name }}
         key: >

diff --git a/.github/actions/cache/action.yml b/.github/actions/cache/action.yml
@@ -0,0 +1,45 @@
+---
+name: cache-decider
+description: Decide which cache to use
+inputs:
+  key:
+    required: false
+    type: string
+    description: The onedir package name to create
+    default: salt
+  path:
+    required: true
+    type: string
+    description: path to contents to cache
+
+outputs:
+  cache-hit:
+    value: ${{ steps.s3-cache.outputs.cache-hit || steps.github-cache.outputs.cache-hit }}
+
+runs:
+  using: composite
+
+  steps:
+    - name: Configure AWS Credentials to access cache bucket
+      if: ${{ env.USE_S3_CACHE == 'true' }}
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        aws-region: us-west-2
+
+    - name: Cache Deps Onedir Package Directory
+      if: ${{ env.USE_S3_CACHE == 'true' }}
+      id: s3-cache
+      env:
+        RUNS_ON_S3_BUCKET_CACHE: "salt-project-test-salt-github-actions-s3-cache"
+      uses: runs-on/cache@v4
+      with:
+        path: ${{ inputs.path }}
+        key: ${{ inputs.key }}
+
+    - name: Cache Deps Onedir Package Directory
+      if: ${{ env.USE_S3_CACHE != 'true' }}
+      id: github-cache
+      uses: ./.github/actions/cache
+      with:
+        path: ${{ inputs.path }}
+        key: ${{ inputs.key }}
diff --git a/.github/actions/cached-virtualenv/action.yml b/.github/actions/cached-virtualenv/action.yml
@@ -54,7 +54,7 @@ runs:
 
     - name: Cache VirtualEnv
       id: cache-virtualenv
-      uses: actions/cache@v3.3.1
+      uses: ./.github/actions/cache
       with:
         key: ${{ steps.setup-cache-key.outputs.cache-key }}
         path: ${{ steps.virtualenv-path.outputs.venv-path }}

diff --git a/.github/actions/setup-actionlint/action.yml b/.github/actions/setup-actionlint/action.yml
@@ -15,7 +15,7 @@ runs:
   steps:
 
     - name: Cache actionlint Binary
-      uses: actions/cache@v3.3.1
+      uses: ./.github/actions/cache
       with:
         path: /usr/local/bin/actionlint
         key: ${{ inputs.cache-seed }}|${{ runner.os }}|${{ runner.arch }}|actionlint|${{ inputs.version }}

diff --git a/.github/actions/setup-pre-commit/action.yml b/.github/actions/setup-pre-commit/action.yml
@@ -36,7 +36,7 @@ runs:
         ${{ steps.pre-commit-virtualenv.outputs.python-executable }} -m pip install pre-commit==${{ inputs.version }}
 
     - name: Cache Pre-Commit Hooks
-      uses: actions/cache@v3.3.1
+      uses: ./.github/actions/cache
       id: pre-commit-hooks-cache
       with:
         key: ${{ steps.pre-commit-virtualenv.outputs.cache-key }}|${{ inputs.version }}|${{ hashFiles('.pre-commit-config.yaml') }}

diff --git a/.github/actions/setup-python-tools-scripts/action.yml b/.github/actions/setup-python-tools-scripts/action.yml
@@ -50,7 +50,7 @@ runs:
         cache-seed: tools|${{ steps.venv-hash.outputs.venv-hash }}
 
     - name: Restore Python Tools Virtualenvs Cache
-      uses: actions/cache@v3.3.1
+      uses: ./.github/actions/cache
       with:
         path: ${{ inputs.cwd }}/.tools-venvs
         key: ${{ inputs.cache-prefix }}|${{ steps.venv-hash.outputs.venv-hash }}

diff --git a/.github/actions/setup-relenv/action.yml b/.github/actions/setup-relenv/action.yml
@@ -45,7 +45,7 @@ runs:
         python3 -m pip install relenv==${{ inputs.version }}
 
     - name: Cache Relenv Data Directory
-      uses: actions/cache@v3.3.1
+      uses: ./.github/actions/cache
       with:
         path: ${{ github.workspace }}/.relenv
         key: ${{ inputs.cache-seed }}|relenv|${{ inputs.version }}|${{ inputs.python-version }}|${{ inputs.platform }}|${{ inputs.arch }}

diff --git a/.github/actions/setup-shellcheck/action.yml b/.github/actions/setup-shellcheck/action.yml
@@ -15,7 +15,7 @@ runs:
   steps:
 
     - name: Cache shellcheck Binary
-      uses: actions/cache@v3.3.1
+      uses: ./.github/actions/cache
       with:
         path: /usr/local/bin/shellcheck
         key: ${{ inputs.cache-seed }}|${{ runner.os }}|${{ runner.arch }}|shellcheck|${{ inputs.version }}

diff --git a/.github/workflows/build-deps-ci-action.yml b/.github/workflows/build-deps-ci-action.yml
@@ -77,7 +77,7 @@ jobs:
 
       - name: Cache nox.linux.${{ matrix.arch }}.tar.* for session ${{ inputs.nox-session }}
         id: nox-dependencies-cache
-        uses: actions/cache@v3.3.1
+        uses: ./.github/actions/cache
         with:
           path: nox.linux.${{ matrix.arch }}.tar.*
           key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|linux|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ inputs.nox-archive-hash }}
@@ -184,7 +184,7 @@ jobs:
 
       - name: Cache nox.macos.${{ matrix.arch }}.tar.* for session ${{ inputs.nox-session }}
         id: nox-dependencies-cache
-        uses: actions/cache@v3.3.1
+        uses: ./.github/actions/cache
         with:
           path: nox.macos.${{ matrix.arch }}.tar.*
           key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|macos|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ inputs.nox-archive-hash }}
@@ -273,7 +273,7 @@ jobs:
 
       - name: Cache nox.windows.${{ matrix.arch }}.tar.* for session ${{ inputs.nox-session }}
         id: nox-dependencies-cache
-        uses: actions/cache@v3.3.1
+        uses: ./.github/actions/cache
         with:
           path: nox.windows.${{ matrix.arch }}.tar.*
           key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|windows|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ inputs.nox-archive-hash }}

diff --git a/.github/workflows/build-deps-onedir.yml b/.github/workflows/build-deps-onedir.yml
@@ -54,6 +54,8 @@ jobs:
       - self-hosted
       - linux
       - ${{ matrix.arch }}
+    env:
+      USE_S3_CACHE: 'true'
     steps:
 
       - name: "Throttle Builds"
@@ -96,7 +98,8 @@ jobs:
         arch: ${{ inputs.github-enterprise-runners == true && fromJSON('["x86_64", "arm64"]') || fromJSON('["x86_64"]') }}
     runs-on:
       - ${{ matrix.arch == 'arm64' && 'macos-13-xlarge' || 'macos-12' }}
-
+    env:
+      USE_S3_CACHE: 'false'
     steps:
 
       - name: "Throttle Builds"
@@ -145,6 +148,8 @@ jobs:
           - x86
           - amd64
     runs-on: windows-latest
+    env:
+      USE_S3_CACHE: 'false'
     steps:
 
       - name: "Throttle Builds"

diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml
@@ -69,7 +69,7 @@ jobs:
 
       - name: Cache Python Tools Docs Virtualenv
         id: tools-venvs-dependencies-cache
-        uses: actions/cache@v3.3.1
+        uses: ./.github/actions/cache
         with:
           path: .tools-venvs/docs
           key: ${{ inputs.cache-seed }}|${{ github.workflow }}|${{ github.job }}|tools-venvs|${{ steps.python-tools-scripts.outputs.version }}|docs|${{ steps.get-python-version.outputs.version }}|${{ hashFiles('requirements/**/docs.txt') }}

diff --git a/.github/workflows/templates/build-deps-ci-action.yml.jinja b/.github/workflows/templates/build-deps-ci-action.yml.jinja
@@ -77,7 +77,7 @@ jobs:
 
       - name: Cache nox.linux.${{ matrix.arch }}.tar.* for session ${{ inputs.nox-session }}
         id: nox-dependencies-cache
-        uses: actions/cache@v3.3.1
+        uses: ./.github/actions/cache
         with:
           path: nox.linux.${{ matrix.arch }}.tar.*
           key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|linux|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ inputs.nox-archive-hash }}
@@ -184,7 +184,7 @@ jobs:
 
       - name: Cache nox.macos.${{ matrix.arch }}.tar.* for session ${{ inputs.nox-session }}
         id: nox-dependencies-cache
-        uses: actions/cache@v3.3.1
+        uses: ./.github/actions/cache
         with:
           path: nox.macos.${{ matrix.arch }}.tar.*
           key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|macos|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ inputs.nox-archive-hash }}
@@ -275,7 +275,7 @@ jobs:
 
       - name: Cache nox.windows.${{ matrix.arch }}.tar.* for session ${{ inputs.nox-session }}
         id: nox-dependencies-cache
-        uses: actions/cache@v3.3.1
+        uses: ./.github/actions/cache
         with:
           path: nox.windows.${{ matrix.arch }}.tar.*
           key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|windows|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ inputs.nox-archive-hash }}