refactor(ci): include node versions, break up build

This commit heavily refactors the CI build pipeline to separate different build steps to improve build times and caching. It *should* be easier to see the dependency between the builds: - splicer (jit/AOT independent) - StarlingMonkey JIT - StarlingMonkey AOT (Weval) Along with these builds, JIT/AOT tests that should come after can pull the relevant cached artifacts. Signed-off-by: Victor Adossi <[email protected]>
bytecodealliance · Nov 9, 2024 · 210d9b0 · 210d9b0
1 parent ae4755f
commit 210d9b0
Showing 1 changed file with 319 additions and 63 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -16,72 +16,328 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  test:
-    name: Test
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
+      - name: Install Rust
+        run: |
+          rustup update stable
+          rustup default stable
+          rustup component add rustfmt
+      - name: Format source code
+        run: cargo fmt -- --check
+
+  #########
+  # Build #
+  #########
+
+  build-splicer:
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        config:
-          - { os: ubuntu-latest, build: "npm run build", test: "npm run test", cacheDirectory: "build-release", cacheKeySuffix: "", enableAot: "0"  }
-          - { os: ubuntu-latest, build: "npm run build:weval", test: "npm run test:weval", cacheDirectory: "build-release-weval", cacheKeySuffix: "-weval", enableAot: "1" }
-    runs-on: ${{ matrix.config.os }}
+        node-version:
+          - '20'
+          - latest
     steps:
-    - uses: actions/checkout@v2
-      with:
-        submodules: recursive
-
-    - name: Install Rust Toolchain
-      run: |
-        rustup toolchain install 1.77.1
-        rustup target add wasm32-wasi --toolchain 1.77.1
-        rustup target add wasm32-wasi
-
-    - uses: actions/setup-node@v2
-      with:
-        node-version: '20'
-
-    - name: Cache Rust dependencies
-      uses: actions/cache@v3
-      id: rust-build
-      with:
-        path: target
-        key: engine-build-${{ hashFiles('Cargo.lock', 'crates/spidermonkey-embedding-splicer/src/**/*.rs') }}
-
-    - name: Install NPM packages
-      run: npm install
-
-    - name: Get StarlingMonkey Commit
-      id: starlingmonkey-commit
-      run: echo "STARLINGMONKEY_HASH=$(git submodule status | head -c9 | tail -c8)" >> "$GITHUB_OUTPUT"
-
-    - name: Cache StarlingMonkey
-      uses: actions/cache@v3
-      id: starlingmonkey-build
-      with:
-        path: ${{matrix.config.cacheDirectory}} 
-        key: engine-build${{matrix.config.cacheKeySuffix}}-${{ steps.starlingmonkey-commit.outputs.STARLINGMONKEY_HASH }}
-
-    - name: Build
-      run: ${{matrix.config.build}}
-
-    - name: Test
-      run: ${{matrix.config.test}}
-
-    - name: Cache Example build
-      uses: actions/cache@v3
-      with:
-        path: example/target
-        key: engine-cargo-${{ hashFiles('example/src/main.rs', 'example/Cargo.lock', 'example/hello.wit') }}
-
-    - name: Test Example
-      run: cd example && ENABLE_AOT=${{matrix.config.enableAot}} npm run build && ./test.sh
-
-  rustfmt:
-    name: Rustfmt
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
+
+      - name: Install Rust Toolchain
+        run: |
+          rustup toolchain install 1.77.1
+          rustup target add wasm32-wasi --toolchain 1.77.1
+          rustup target add wasm32-wasi
+
+      - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # 4.1.0
+        with:
+          node-version: ${{matrix.node-version}}
+
+      - name: Install NPM packages
+        run: npm install
+
+      - name: Cache Splicer build
+        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # 4.1.2
+        id: splicer-build
+        with:
+          key: generated-splicer-node-${{matrix.node-version}}-${{ hashFiles('Cargo.lock', 'crates/spidermonkey-embedding-splicer/src/**/*.rs') }}
+          path: |
+            lib/spidermonkey-embedding-splicer.core2.wasm
+            lib/spidermonkey-embedding-splicer.core.wasm
+            lib/spidermonkey-embedding-splicer.d.ts
+            lib/spidermonkey-embedding-splicer.js
+            target
+
+      - name: Build splicer
+        if: steps.splicer-build.outputs.cache-hit != 'true'
+        run: |
+          make lib/spidermonkey-embedding-splicer.js
+
+  build-jit:
+    runs-on: ubuntu-latest
+    needs:
+      - build-splicer
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version:
+          - '20'
+          - latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
+        with:
+          submodules: recursive
+
+      - name: Get StarlingMonkey Commit
+        id: starlingmonkey-commit
+        run: echo "STARLINGMONKEY_HASH=$(git submodule status | head -c9 | tail -c8)" >> "$GITHUB_OUTPUT"
+
+      - name: Install Rust Toolchain
+        run: |
+          rustup toolchain install 1.77.1
+          rustup target add wasm32-wasi --toolchain 1.77.1
+          rustup target add wasm32-wasi
+
+      - name: Restore Embedding Splicer from cache
+        uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # 4.1.2
+        id: splicer-build
+        with:
+          key: generated-splicer-node-${{matrix.node-version}}-${{ hashFiles('Cargo.lock', 'crates/spidermonkey-embedding-splicer/src/**/*.rs') }}
+          path: |
+            lib/spidermonkey-embedding-splicer.core2.wasm
+            lib/spidermonkey-embedding-splicer.core.wasm
+            lib/spidermonkey-embedding-splicer.d.ts
+            lib/spidermonkey-embedding-splicer.js
+            target
+
+      - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # 4.1.0
+        with:
+          node-version: ${{matrix.node-version}}
+
+      - name: Install NPM packages
+        run: npm install
+
+      - name: Cache StarlingMonkey
+        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # 4.1.2
+        id: starlingmonkey-jit
+        with:
+          key: generated-starlingmonkey-jit-node-${{matrix.node-version}}-${{ steps.starlingmonkey-commit.outputs.STARLINGMONKEY_HASH }}
+          lookup-only: 'true'
+          path: |
+            lib/starlingmonkey_embedding.wasm
+            build-release
+
+      - name: Build ComponentizeJS
+        if: steps.starlingmonkey-jit.outputs.cache-hit != 'true'
+        run: |
+          make release
+          make lib/starlingmonkey_embedding.wasm
+
+      - uses: actions/upload-artifact@v4
+        if: steps.starlingmonkey-jit.outputs.cache-hit != 'true'
+        with:
+          name: starlingmonkey-jit-node-${{matrix.node-version}}-${{ steps.starlingmonkey-commit.outputs.STARLINGMONKEY_HASH }}
+          path: |
+            lib/starlingmonkey_embedding.wasm
+            build-release
+
+  build-aot:
     runs-on: ubuntu-latest
+    needs:
+      - build-splicer
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version:
+          - '20'
+          - latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Install Rust
-      run: rustup update stable && rustup default stable && rustup component add rustfmt
-    - name: Format source code
-      run: cargo fmt -- --check
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
+        with:
+          submodules: recursive
+
+      - name: Get StarlingMonkey Commit
+        id: starlingmonkey-commit
+        run: echo "STARLINGMONKEY_HASH=$(git submodule status | head -c9 | tail -c8)" >> "$GITHUB_OUTPUT"
+
+      - name: Install Rust Toolchain
+        run: |
+          rustup toolchain install 1.77.1
+          rustup target add wasm32-wasi --toolchain 1.77.1
+          rustup target add wasm32-wasi
+
+      - name: Restore Embedding Splicer from cache
+        uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # 4.1.2
+        id: splicer-build
+        with:
+          key: generated-splicer-node-${{matrix.node-version}}-${{ hashFiles('Cargo.lock', 'crates/spidermonkey-embedding-splicer/src/**/*.rs') }}
+          path: |
+            lib/spidermonkey-embedding-splicer.core2.wasm
+            lib/spidermonkey-embedding-splicer.core.wasm
+            lib/spidermonkey-embedding-splicer.d.ts
+            lib/spidermonkey-embedding-splicer.js
+            target
+
+      - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # 4.1.0
+        with:
+          node-version: ${{matrix.node-version}}
+
+      - name: Install NPM packages
+        run: npm install
+
+      - name: Cache StarlingMonkey (Weval)
+        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # 4.1.2
+        id: starlingmonkey-aot
+        with:
+          key: generated-starlingmonkey-aot-node-${{matrix.node-version}}-${{ steps.starlingmonkey-commit.outputs.STARLINGMONKEY_HASH }}
+          lookup-only: 'true'
+          path: |
+            lib/starlingmonkey_embedding_weval.wasm
+            lib/starlingmonkey_ics.wevalcache
+            build-release-weval
+
+      - name: Build Weval
+        if: steps.starlingmonkey-aot.outputs.cache-hit != 'true'
+        run: |
+          make release-weval
+          make lib/starlingmonkey_embedding_weval.wasm
+          make lib/starlingmonkey_ics.wevalcache
+
+      - uses: actions/upload-artifact@v4
+        if: steps.starlingmonkey-aot.outputs.cache-hit != 'true'
+        with:
+          name: starlingmonkey-aot-node-${{matrix.node-version}}-${{ steps.starlingmonkey-commit.outputs.STARLINGMONKEY_HASH }}
+          path: |
+            lib/starlingmonkey_embedding.wasm
+            build-release
+
+  ########
+  # Test #
+  ########
+
+  test-jit:
+    runs-on: ubuntu-latest
+    needs:
+      - build-jit
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version:
+          - '20'
+          - latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
+
+      - name: Get StarlingMonkey Commit
+        id: starlingmonkey-commit
+        run: echo "STARLINGMONKEY_HASH=$(git submodule status | head -c9 | tail -c8)" >> "$GITHUB_OUTPUT"
+
+      - name: Restore Embedding Splicer from cache
+        uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # 4.1.2
+        id: splicer-build
+        with:
+          key: generated-splicer-node-${{matrix.node-version}}-${{ hashFiles('Cargo.lock', 'crates/spidermonkey-embedding-splicer/src/**/*.rs') }}
+          path: |
+            lib/spidermonkey-embedding-splicer.core2.wasm
+            lib/spidermonkey-embedding-splicer.core.wasm
+            lib/spidermonkey-embedding-splicer.d.ts
+            lib/spidermonkey-embedding-splicer.js
+            target
+
+      - name: Restore StarlingMonkey from cache
+        uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # 4.1.2
+        id: starlingmonkey-jit
+        with:
+          key: generated-starlingmonkey-jit-node-${{matrix.node-version}}-${{ steps.starlingmonkey-commit.outputs.STARLINGMONKEY_HASH }}
+          path: |
+            lib/starlingmonkey_embedding.wasm
+            target
+
+      - uses: actions/download-artifact@v4
+        if: steps.starlingmonkey-jit.outputs.cache-hit != 'true'
+        with:
+          name: starlingmonkey-jit-node-${{matrix.node-version}}-${{ steps.starlingmonkey-commit.outputs.STARLINGMONKEY_HASH }}
+
+      - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # 4.1.0
+        with:
+          node-version: ${{matrix.node-version}}
+
+      - name: Install NPM packages
+        run: npm install
+
+      - name: Test
+        run: npm run test
+
+      - name: Cache Example build
+        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # 4.1.2
+        with:
+          path: example/target
+          key: generated-example-jit-cargo-${{ hashFiles('example/src/main.rs', 'example/Cargo.lock', 'example/hello.wit') }}
+
+      - name: Test Example
+        run: cd example && npm run build && ./test.sh
+
+  test-aot:
+    runs-on: ubuntu-latest
+    needs:
+      - build-aot
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version:
+          - '20'
+          - latest
+    env:
+      ENABLE_AOT: "1"
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
+
+      - name: Get StarlingMonkey Commit
+        id: starlingmonkey-commit
+        run: echo "STARLINGMONKEY_HASH=$(git submodule status | head -c9 | tail -c8)" >> "$GITHUB_OUTPUT"
+
+      - name: Restore Embedding Splicer from cache
+        uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # 4.1.2
+        id: splicer-build
+        with:
+          key: generated-splicer-node-${{matrix.node-version}}-${{ hashFiles('Cargo.lock', 'crates/spidermonkey-embedding-splicer/src/**/*.rs') }}
+          path: |
+            lib/spidermonkey-embedding-splicer.core2.wasm
+            lib/spidermonkey-embedding-splicer.core.wasm
+            lib/spidermonkey-embedding-splicer.d.ts
+            lib/spidermonkey-embedding-splicer.js
+            target
+
+      - name: Restore StarlingMonkey (Weval) from cache
+        uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # 4.1.2
+        id: starlingmonkey-aot
+        with:
+          key: generated-starlingmonkey-aot-node-${{matrix.node-version}}-${{ steps.starlingmonkey-commit.outputs.STARLINGMONKEY_HASH }}
+          path: |
+            lib/starlingmonkey_embedding_weval.wasm
+            lib/starlingmonkey_ics.wevalcache
+            target
+
+      - uses: actions/download-artifact@v4
+        if: steps.starlingmonkey-aot.outputs.cache-hit != 'true'
+        with:
+          name: starlingmonkey-aot-node-${{matrix.node-version}}-${{ steps.starlingmonkey-commit.outputs.STARLINGMONKEY_HASH }}
+
+      - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # 4.1.0
+        with:
+          node-version: ${{matrix.node-version}}
+
+      - name: Install NPM packages
+        run: npm install
+
+      - name: Test
+        run: npm run test:weval
+
+      - name: Cache Example build
+        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # 4.1.2
+        with:
+          path: example/target
+          key: generated-example-aot-cargo-${{ hashFiles('example/src/main.rs', 'example/Cargo.lock', 'example/hello.wit') }}
+
+      - name: Test Example
+        run: cd example && npm run build && ./test.sh