Add integration tests for resource explorer refresh and bundle job re…

…source run (#1108) ## Changes * Add e2e integration test for refreshing a bundle when bundle file changes * Add e2e integration test for running a bundle job resource * Add a common prefix to all resources created by bundle tests to ease cleanup. ## Tests
databricks · Apr 23, 2024 · 5177b8d · 5177b8d
1 parent f00ebd2
commit 5177b8d
Show file tree

Hide file tree

Showing 20 changed files with 907 additions and 280 deletions.
diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
@@ -26,9 +26,15 @@ jobs:
             cli_arch: ${{ matrix.arch.cli_arch }}
         secrets: inherit
 
+    validate-tests:
+        runs-on: ubuntu-latest
+        needs: ["run-tests"]
+        steps:
+            - run: echo "Tests successfull"
+              shell: bash
+
     package:
-        name: Package VSIX
-        needs: "run-tests"
+        name: Package Arm64 VSIX
         runs-on: "macos-latest"
         steps:
             - uses: actions/checkout@v3
@@ -53,7 +59,7 @@ jobs:
             - run: mkdir -p packages/databricks-vscode/artifacts
 
             - name: Build VSIX
-              run: yarn package -o artifacts -t darwin-x64
+              run: yarn package -o artifacts -t darwin-arm64
               working-directory: packages/databricks-vscode
 
             - name: Upload artifacts

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -17,8 +17,8 @@ on:
         type: string
 
 jobs:
-  test-extension:
-    name: Test VSCode Extension
+  unit-test-extension:
+    name: Run unit tests
     runs-on: ${{ inputs.os }}
     environment:
       name: azure-prod-usr
@@ -43,12 +43,6 @@ jobs:
           node-version: ${{ inputs.node-version }}
           cache: "yarn"
 
-      - name: Cache wdio
-        uses: actions/cache@v3
-        with:
-          path: /tmp/wdio-vscode-service
-          key: ${{ inputs.cli_arch }}-${{ inputs.vscode-version }}-wdio
-
       - name: Cache VSCode unit test runner
         uses: actions/cache@v3
         with:
@@ -76,18 +70,143 @@ jobs:
           run: yarn run test:cov
           working-directory: packages/databricks-vscode
 
-      - name: Integration Tests
-        run: yarn run test:integ:prepare && yarn run test:integ:run
+  list-integration-test-specs:
+    name: Generate integration test matrix
+    runs-on: ${{ inputs.os }}
+    outputs:
+      specs: ${{ steps.generate-matrix.outputs.specs }}
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Use Node.js ${{ inputs.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ inputs.node-version }}
+          cache: "yarn"
+
+      - run: yarn install --immutable
+
+      - name: Generate matrix
+        id: generate-matrix
+        run: |
+          echo "specs=$(yarn ts-node scripts/list_integration_tests.ts)"
+          echo "specs=$(yarn ts-node scripts/list_integration_tests.ts)" >> $GITHUB_OUTPUT
         working-directory: packages/databricks-vscode
 
-      - name: Integration Tests SDK wrappers
-        run: yarn run test:integ:sdk
+  extension-integration-tests:
+    name: Run extension e2e integration tests
+    needs: [list-integration-test-specs]
+    strategy:
+      fail-fast: false
+      matrix:
+        spec: ${{fromJson(needs.list-integration-test-specs.outputs.specs)}}
+
+    runs-on: ${{ inputs.os }}
+    environment:
+      name: azure-prod-usr
+
+    env:
+      VSCODE_TEST_VERSION: ${{ inputs.vscode-version }}
+      DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
+      DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
+      TEST_DEFAULT_CLUSTER_ID: ${{ secrets.TEST_VSCODE_CLUSTER_ID }}
+      CLI_ARCH: ${{ inputs.cli_arch }}
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Use Node.js ${{ inputs.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ inputs.node-version }}
+          cache: "yarn"
+
+      - name: Get vscode engine
+        id: vscode-engine
+        run: |
+          echo "vscode=$(cat package.json | jq .engines.vscode | tr -d '^"')"
+          echo "vscode=$(cat package.json | jq .engines.vscode | tr -d '^"')" >> $GITHUB_OUTPUT
+        working-directory: packages/databricks-vscode
+
+      - name: Cache wdio
+        id: wdio-cache
+        uses: actions/cache@v3
+        with:
+          path: packages/databricks-vscode/tmp/wdio-vscode-service
+          key: ${{ inputs.cli_arch }}-${{ steps.vscode-engine.outputs.vscode }}-wdio
+
+      - name: mkdir packages/databricks-vscode/tmp/wdio-vscode-service
+        run: mkdir -p packages/databricks-vscode/tmp/wdio-vscode-service
+        if: steps.wdio-cache.outputs.cache-hit != 'true'
+
+      - run: yarn install --immutable
+
+      - name: Fetching Databricks CLI
+        run: yarn run package:cli:fetch
+        working-directory: packages/databricks-vscode
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Building packages
+        run: yarn run build
+
+      - name: Integration Tests
+        run: yarn run test:integ:extension --spec ${{ matrix.spec.path }}
         working-directory: packages/databricks-vscode
 
       - name: Upload test logs
         if: always()
         continue-on-error: true
         uses: actions/upload-artifact@v3
         with:
-          name: test-logs ${{ join(inputs.*, ' - ') }} - ${{ github.event_name }}
+          name: test-logs ${{ join(inputs.*, ' - ') }} - ${{ matrix.spec.baseName }} - ${{ github.event_name }}
           path: packages/databricks-vscode/logs
+
+  sdk-wrappers-integration-tests:
+    name: Run integration tests for SDK wrappers
+    runs-on: ${{ inputs.os }}
+    environment:
+      name: azure-prod-usr
+
+    env:
+      VSCODE_TEST_VERSION: ${{ inputs.vscode-version }}
+      DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
+      DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
+      TEST_DEFAULT_CLUSTER_ID: ${{ secrets.TEST_VSCODE_CLUSTER_ID }}
+      CLI_ARCH: ${{ inputs.cli_arch }}
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Use Node.js ${{ inputs.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ inputs.node-version }}
+          cache: "yarn"
+
+      - run: yarn install --immutable
+
+      - name: Fetching Databricks CLI
+        run: yarn run package:cli:fetch
+        working-directory: packages/databricks-vscode
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Building packages
+        run: yarn run build
+
+      - name: Integration Tests SDK wrappers
+        run: yarn run test:integ:sdk
+        working-directory: packages/databricks-vscode
diff --git a/packages/databricks-vscode/.gitignore b/packages/databricks-vscode/.gitignore
@@ -5,6 +5,7 @@ src/test/e2e/workspace/
 extension/
 .pytest_cache/
 .build/
+**/tmp/**
 
 # Telemetry file, automatically generated by packages/databricks-vscode/scripts/generateTelemetry.ts
 telemetry.json
diff --git a/packages/databricks-vscode/package.json b/packages/databricks-vscode/package.json
@@ -802,9 +802,9 @@
         "test:lint": "eslint src --ext ts && prettier . -c",
         "test:unit": "yarn run build && node ./out/test/runTest.js",
         "test:integ:prepare": "yarn run package",
-        "test:integ:run": "wdio run src/test/e2e/wdio.conf.ts",
+        "test:integ:extension": "yarn run test:integ:prepare && wdio run src/test/e2e/wdio.conf.ts",
         "test:integ:sdk": "ts-mocha --type-check 'src/sdk-extensions/**/*.integ.ts'",
-        "test:integ": "yarn run test:integ:prepare && yarn run test:integ:run && yarn run test:integ:sdk",
+        "test:integ": "yarn run test:integ:extension && yarn run test:integ:sdk",
         "test:cov": "nyc yarn run test:unit",
         "test": "yarn run test:lint && yarn run test:unit",
         "clean": "rm -rf node_modules out .vscode-test",

diff --git a/packages/databricks-vscode/scripts/list_integration_tests.ts b/packages/databricks-vscode/scripts/list_integration_tests.ts
@@ -0,0 +1,28 @@
+import {glob} from "glob";
+import path from "path";
+
+function toUnixPath(path: string) {
+    if (process.platform === "win32") {
+        return path.replace(/\\/g, "/");
+    }
+    return path;
+}
+
+const integrationTests = glob
+    .globSync(
+        toUnixPath(
+            path.join(process.cwd(), "src", "test", "e2e", "**", "*.e2e.ts")
+        ),
+        {
+            nocase: process.platform === "win32",
+        }
+    )
+    .map((testPath) => {
+        return {
+            path: toUnixPath(path.relative(process.cwd(), testPath)),
+            baseName: path.basename(testPath, ".e2e.ts"),
+        };
+    });
+
+// eslint-disable-next-line no-console
+console.log(JSON.stringify(integrationTests));
diff --git a/packages/databricks-vscode/src/bundle/BundleFileSet.ts b/packages/databricks-vscode/src/bundle/BundleFileSet.ts
@@ -30,9 +30,7 @@ export async function getSubProjects(root: Uri) {
         .map((rootFile) => {
             const dirname = path.dirname(path.normalize(rootFile));
             const absolute = Uri.file(dirname);
-            const relative = Uri.file(
-                absolute.fsPath.replace(normalizedRoot, "")
-            );
+            const relative = path.relative(normalizedRoot, dirname);
             return {absolute, relative};
         })
         .filter(({absolute}) => {

diff --git a/packages/databricks-vscode/src/bundle/BundleInitWizard.ts b/packages/databricks-vscode/src/bundle/BundleInitWizard.ts
@@ -20,14 +20,14 @@ import {writeFile, mkdir} from "fs/promises";
 import path from "path";
 
 export async function promptToOpenSubProjects(
-    projects: {absolute: Uri; relative: Uri}[],
+    projects: {absolute: Uri; relative: string}[],
     authProvider?: AuthProvider
 ) {
     type OpenProjectItem = QuickPickItem & {uri?: Uri};
     const items: OpenProjectItem[] = projects.map((project) => {
         return {
             uri: project.absolute,
-            label: project.relative.fsPath,
+            label: project.relative,
             detail: project.absolute.fsPath,
         };
     });

diff --git a/packages/databricks-vscode/src/bundle/BundleProjectManager.ts b/packages/databricks-vscode/src/bundle/BundleProjectManager.ts
@@ -33,7 +33,7 @@ export class BundleProjectManager {
     private projectServicesReady = false;
     private projectServicesMutex = new Mutex();
 
-    private subProjects?: {relative: Uri; absolute: Uri}[];
+    private subProjects?: {relative: string; absolute: Uri}[];
     private legacyProjectConfig?: ProjectConfigFile;
 
     constructor(

diff --git a/packages/databricks-vscode/src/test/e2e/auth.e2e.ts b/packages/databricks-vscode/src/test/e2e/auth.e2e.ts
@@ -1,43 +1,24 @@
 import assert from "node:assert";
-import * as fs from "fs/promises";
 import {
     dismissNotifications,
     waitForInput,
     getViewSection,
     waitForLogin,
-    clearBundleConfig,
-    createBasicBundleConfig,
-} from "./utils.ts";
+} from "./utils/commonUtils.ts";
 import {CustomTreeSection} from "wdio-vscode-service";
-
-let cfgPath: string;
-let cfgContent: Buffer;
+import {
+    getBasicBundleConfig,
+    writeRootBundleConfig,
+} from "./utils/dabsFixtures.ts";
 
 describe("Configure Databricks Extension", async function () {
     this.timeout(3 * 60 * 1000);
 
-    before(async function () {
-        assert(process.env.WORKSPACE_PATH, "WORKSPACE_PATH doesn't exist");
+    it("should open VSCode and dismiss notifications", async function () {
         assert(
             process.env.DATABRICKS_CONFIG_FILE,
             "DATABRICKS_CONFIG_FILE doesn't exist"
         );
-        cfgPath = process.env.DATABRICKS_CONFIG_FILE;
-        cfgContent = await fs.readFile(cfgPath);
-    });
-
-    after(async function () {
-        try {
-            await clearBundleConfig();
-            if (cfgContent) {
-                await fs.writeFile(cfgPath, cfgContent);
-            }
-        } catch (e) {
-            console.error(e);
-        }
-    });
-
-    it("should open VSCode and dismiss notifications", async function () {
         const workbench = await browser.getWorkbench();
         const title = await workbench.getTitleBar().getTitle();
         assert(
@@ -60,7 +41,11 @@ describe("Configure Databricks Extension", async function () {
     });
 
     it("should automatically login after detecting bundle configuration", async () => {
-        await createBasicBundleConfig();
+        assert(process.env.WORKSPACE_PATH, "WORKSPACE_PATH doesn't exist");
+        await writeRootBundleConfig(
+            getBasicBundleConfig(),
+            process.env.WORKSPACE_PATH
+        );
         await waitForLogin("DEFAULT");
     });