Skip to content

Commit

Permalink
[ci] more robustness in tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@DavyLandman
DavyLandman committed Sep 29, 2023
1 parent 57e6037 commit 6cfe9d9
Show file tree
Hide file tree
Showing 5 changed files with 63 additions and 65 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ jobs:
working-directory: ./rascal-vscode-extension
env:
DELAY_FACTOR: 2
_JAVA_OPTIONS: '-Xmx7G'
_JAVA_OPTIONS: '-Xmx5G' # we have 16gb of memory, make sure LSP, REPL & DSL-LSP can start
run: xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' npx extest setup-and-run out/test/vscode-suite/*.test.js --storage /tmp/uitests

- name: Upload Screenshots
Expand Down
18 changes: 14 additions & 4 deletions rascal-vscode-extension/src/test/vscode-suite/dsl.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
*/

import { VSBrowser, WebDriver, Workbench } from 'vscode-extension-tester';
import { IDEOperations, RascalREPL, TestWorkspace } from './utils';
import { Delays, IDEOperations, RascalREPL, TestWorkspace } from './utils';
import { expect } from 'chai';
import * as fs from 'fs/promises';

Expand All @@ -38,17 +38,22 @@ describe('DSL', function () {
let ide : IDEOperations;
let picoFileBackup: Buffer;

this.timeout(120_000);
this.timeout(Delays.extremelySlow * 2);


async function loadPico() {
const repl = new RascalREPL(bench, driver);
console.log("Starting REPL");
await repl.start();
console.log("execute import");
await repl.execute("import demo::lang::pico::LanguageServer;");
console.log("execute main");
await repl.execute("main();");
expect(repl.lastOutput).is.equal("ok");
const statusBarCheck = driver.wait(ide.statusContains("Pico"), 20_000, "Pico DSL should start loading");
const statusBarCheck = driver.wait(ide.statusContains("Pico"), Delays.verySlow, "Pico DSL should start loading");
console.log("terminate repl");
await repl.terminate();
console.log("wait for status bar to appear");
await statusBarCheck; // now we wait for pico to finish loading
}

Expand All @@ -61,17 +66,22 @@ describe('DSL', function () {
ide = new IDEOperations(browser, bench);
await ide.load();
await loadPico();
console.log("Done loading pico");
picoFileBackup = await fs.readFile(TestWorkspace.picoFile);
});

afterEach(async function () {
if (this.test?.title) {
await ide.screenshot(this.test?.title);
await ide.screenshot("DSL-" + this.test?.title);
}
await ide.cleanup();
await fs.writeFile(TestWorkspace.picoFile, picoFileBackup);
});

after(async function() {
await ide.screenshot("DSL tests");
});

it("have highlighting and parse errors", async function () {
const editor = await ide.openModule(TestWorkspace.picoFile);
await ide.hasSyntaxHighlighting(editor);
Expand Down
24 changes: 9 additions & 15 deletions rascal-vscode-extension/src/test/vscode-suite/ide.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
import { EditorView, TextEditor, VSBrowser, ViewSection, WebDriver, Workbench } from 'vscode-extension-tester';
import * as path from 'path';
import * as fs from 'fs/promises';
import { IDEOperations, TestWorkspace, sleep } from './utils';
import { Delays, IDEOperations, TestWorkspace, ignoreFails, sleep } from './utils';


const protectFiles = [TestWorkspace.mainFile, TestWorkspace.libFile, TestWorkspace.libCallFile];
Expand All @@ -41,7 +41,7 @@ describe('IDE', function () {
let ide: IDEOperations;
const originalFiles = new Map<string, Buffer>();

this.timeout(100_000);
this.timeout(Delays.extremelySlow * 2);

before(async () => {
browser = VSBrowser.instance;
Expand All @@ -63,15 +63,15 @@ describe('IDE', function () {

afterEach(async function () {
if (this.test?.title) {
await ide.screenshot(this.test?.title);
await ide.screenshot("IDE-" + this.test?.title);
}
await ide.cleanup();
for (const [f, b] of originalFiles) {
await fs.writeFile(f, b);
}
});

async function makeSureRascalModulesAreLoaded(delay = 40_000) {
async function makeSureRascalModulesAreLoaded(delay = Delays.verySlow) {
try {
await ide.openModule(TestWorkspace.mainFile);
let statusBarSeen = false;
Expand Down Expand Up @@ -127,7 +127,7 @@ describe('IDE', function () {
await triggerTypeChecker(editor, TestWorkspace.mainFileTpl, true);
await editor.selectText("println");
await bench.executeCommand("Go to Definition");
await waitForActiveEditor("IO.rsc", 15_000, "IO.rsc should be opened for println");
await waitForActiveEditor("IO.rsc", Delays.extremelySlow, "IO.rsc should be opened for println");
});

it("go to definition works across projects", async () => {
Expand All @@ -140,25 +140,19 @@ describe('IDE', function () {
await triggerTypeChecker(editor, TestWorkspace.libCallFileTpl, true);
await editor.selectText("fib");
await bench.executeCommand("Go to Definition");
await waitForActiveEditor(path.basename(TestWorkspace.libFile), 5_000, "Lib.rsc should be opened for fib");
await waitForActiveEditor(path.basename(TestWorkspace.libFile), Delays.slow, "Lib.rsc should be opened for fib");
});

it("outline works", async () => {
const editor = await ide.openModule(TestWorkspace.mainFile);
await editor.moveCursor(1,1);
const explorer = await (await bench.getActivityBar().getViewControl("Explorer"))!.openView();
await sleep(1000);
await sleep(Delays.fast);
const outline = await explorer.getContent().getSection("Outline") as ViewSection;
await outline.expand();
const mainItem = await driver.wait(async() => {
try {
return outline.findItem("main()", 0);
//return await outline.findElement(By.partialLinkText("main()"));
} catch (_ignored) { return undefined; }
}, 10_000, "Main function should show in the outline");

const mainItem = await driver.wait(async() => ignoreFails(outline.findItem("main()", 0)), Delays.slow, "Main function should show in the outline");
await driver.actions().doubleClick(mainItem!).perform();
await driver.wait(async ()=> (await editor.getCoordinates())[0] === 5, 5_000, "Cursor should have moved to line 6 that contains the println function");
await driver.wait(async ()=> (await editor.getCoordinates())[0] === 5, Delays.normal, "Cursor should have moved to line that contains the println function");
});
});

2 changes: 1 addition & 1 deletion rascal-vscode-extension/src/test/vscode-suite/repl.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ describe('REPL', function () {

afterEach(async function () {
if (this.test?.title) {
await ide.screenshot(this.test?.title);
await ide.screenshot("REPL-"+this.test?.title);
}
await bench.executeCommand("workbench.action.terminal.killAll");
await ide.cleanup();
Expand Down
82 changes: 38 additions & 44 deletions rascal-vscode-extension/src/test/vscode-suite/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,30 +40,39 @@ export class Delays {
private static readonly delayFactor = parseInt(env['DELAY_FACTOR'] ?? "1");
public static readonly fast = sec(1) * this.delayFactor;
public static readonly normal =sec(5) * this.delayFactor;
public static readonly slow =sec(15) * this.delayFactor;
public static readonly slow = sec(15) * this.delayFactor;
public static readonly verySlow =sec(30) * this.delayFactor;
public static readonly extremelySlow =sec(120) * this.delayFactor;
}

function src(project : string, language = 'rascal') { return path.join(project, 'src', 'main', language); }
function target(project : string) { return path.join(project, 'target', 'classes', 'rascal'); }
export class TestWorkspace {
private static get workspacePrefix() { return 'test-workspace'; }
public static get workspaceFile() { return path.join(this.workspacePrefix, 'test.code-workspace'); }
public static get testProject() { return path.join(this.workspacePrefix, 'test-project'); }
public static get libProject() { return path.join(this.workspacePrefix, 'test-lib'); }
public static get mainFile() { return path.join(this.testProject, 'src', 'main', 'rascal', 'Main.rsc'); }
public static get mainFileTpl() { return path.join(this.testProject, 'target', 'classes', 'rascal','Main.tpl'); }
public static get libCallFile() { return path.join(this.testProject, 'src', 'main', 'rascal', 'LibCall.rsc'); }
public static get libCallFileTpl() { return path.join(this.testProject, 'target', 'classes', 'rascal','LibCall.tpl'); }
public static get libFile() { return path.join(this.libProject, 'src', 'main', 'rascal', 'Lib.rsc'); }
public static get libFileTpl() { return path.join(this.libProject, 'target', 'classes', 'rascal','Lib.tpl'); }

public static get picoFile() { return path.join(this.testProject, 'src', 'main', 'pico', 'testing.pico'); }
private static readonly workspacePrefix = 'test-workspace';
public static readonly workspaceFile = path.join(this.workspacePrefix, 'test.code-workspace');
public static readonly testProject = path.join(this.workspacePrefix, 'test-project');
public static readonly libProject = path.join(this.workspacePrefix, 'test-lib');
public static readonly mainFile = path.join(src(this.testProject), 'Main.rsc');
public static readonly mainFileTpl = path.join(target(this.testProject),'Main.tpl');
public static readonly libCallFile = path.join(src(this.testProject), 'LibCall.rsc');
public static readonly libCallFileTpl = path.join(target(this.testProject),'LibCall.tpl');
public static readonly libFile = path.join(src(this.libProject), 'Lib.rsc');
public static readonly libFileTpl = path.join(target(this.libProject),'Lib.tpl');

public static readonly picoFile = path.join(src(this.testProject, 'pico'), 'testing.pico');
}



function ignoreFails<T>(fn : Promise<T>): Promise<T | undefined> {
return fn.catch(() => undefined);
export async function ignoreFails<T>(fn : Promise<T> | undefined): Promise<T | undefined> {
try {
if (!fn) {
return undefined;
}
return await fn;
} catch {
return undefined;
}
}

export class RascalREPL {
Expand Down Expand Up @@ -135,8 +144,8 @@ export class RascalREPL {
}

async terminate() {
await this.execute(":quit", false);
await this.bench.executeCommand("workbench.action.terminal.killAll");
await ignoreFails(this.execute(":quit", false));
await ignoreFails(this.bench.executeCommand("workbench.action.terminal.killAll"));
}
}

Expand All @@ -152,18 +161,19 @@ export class IDEOperations {
}

async load() {
await this.browser.waitForWorkbench(Delays.slow);
await this.browser.openResources(TestWorkspace.workspaceFile);
const center = await this.bench.openNotificationsCenter();
await center.clearAllNotifications();
await center.close();
}

async cleanup() {
await this.revertOpenChanges();
await this.editorView.closeAllEditors();
const center = await this.bench.openNotificationsCenter();
await center.clearAllNotifications();
await center.close();
await ignoreFails(this.revertOpenChanges());
await ignoreFails(this.editorView.closeAllEditors());
const center = await ignoreFails(this.bench.openNotificationsCenter());
await ignoreFails(center?.clearAllNotifications());
await ignoreFails(center?.close());
}

hasElement(editor: TextEditor, selector: Locator, timeout: number, message: string): Promise<WebElement> {
Expand Down Expand Up @@ -194,7 +204,7 @@ export class IDEOperations {
async triggerTypeChecker(editor: TextEditor, { checkName = "Rascal check", waitForFinish = false, timeout = 20_000, tplFile = "" } = {}) {
const lastLine = await editor.getNumberOfLines();
if (tplFile) {
await safeDelete(tplFile);
await ignoreFails(unlink(tplFile));
}
await editor.setTextAtLine(lastLine, await editor.getTextAtLine(lastLine) + " ");
await sleep(50);
Expand All @@ -204,7 +214,7 @@ export class IDEOperations {
let doneChecking = async () => (await this.bench.getStatusBar().getItem(checkName)) === undefined;
if (tplFile) {
const oldDone = doneChecking;
doneChecking = async () => await oldDone() && await fileExists(tplFile);
doneChecking = async () => await oldDone() && (await ignoreFails(stat(tplFile)) !== undefined);
}

await this.driver.wait(doneChecking, timeout, `${checkName} should be finished processing the module`);
Expand All @@ -217,12 +227,10 @@ export class IDEOperations {

statusContains(needle: string): () => Promise<boolean> {
return async () => {
for (const st of await this.bench.getStatusBar().getItems()) {
try {
if ((await st.getText()).includes(needle)) {
return true;
}
} catch (_ignored) { /* sometimes status items get dropped before we can check them */ }
for (const st of await ignoreFails(this.bench.getStatusBar().getItems()) ?? []) {
if ((await ignoreFails(st.getText()))?.includes(needle)) {
return true;
}
}
return false;
};
Expand All @@ -232,17 +240,3 @@ export class IDEOperations {
return this.browser.takeScreenshot(name.replace(/[/\\?%*:|"<>]/g, '-'));
}
}

async function safeDelete(file: string) {
try {
await unlink(file);
} catch (_ignored) { /* ignore deletion errors */ }
}

async function fileExists(file: string): Promise<boolean> {
try {
return await stat(file) !== undefined;
} catch (_ignored) {
return false;
}
}

0 comments on commit 6cfe9d9

Please sign in to comment.