From d0205338bfaabb17216347ebc84553f4edbb0c96 Mon Sep 17 00:00:00 2001 From: BenBoersma Date: Thu, 7 Dec 2023 12:41:01 -0600 Subject: [PATCH 01/10] fixes for upload/download to deal with multinode jenkins runs --- .../src/commands/utils/ScreenshotRequestor.js | 62 ++++++++++++++----- .../wdio-terra-service/WDIOTerraService.js | 12 +++- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/packages/terra-functional-testing/src/commands/utils/ScreenshotRequestor.js b/packages/terra-functional-testing/src/commands/utils/ScreenshotRequestor.js index 7a07db693..bdc5b6ef9 100644 --- a/packages/terra-functional-testing/src/commands/utils/ScreenshotRequestor.js +++ b/packages/terra-functional-testing/src/commands/utils/ScreenshotRequestor.js @@ -55,10 +55,11 @@ class ScreenshotRequestor { /** * Deletes the existing screenshots + * @param {string} referenceName - the name of the reference screenshots file to download */ - async deleteExistingScreenshots() { + async deleteExistingScreenshots(referenceName) { const response = await fetch( - this.serviceUrl, + `${this.serviceUrl}${referenceName}/`, { method: 'DELETE', headers: { @@ -69,7 +70,6 @@ class ScreenshotRequestor { // Allow 404s because that just means the screenshots don't exists yet. ScreenshotRequestor.checkStatus(response, [404]); - logger.info('Existing screenshots deleted from remote repository.'); } /** @@ -80,6 +80,22 @@ class ScreenshotRequestor { fs.removeSync(archiveName); } + /** + * Makes string to return for the screenshots to save and download + * Naming convention is {theme}-{locale}-{browser}-{formfactor} and if provided string is empty does not add it. + * @param {string} locale - the locale to use when downloading + * @param {string} theme - the theme to use when downloading + * @param {string} formFactor - the formFactor to use when downloading + * @param {string} browser - the browser to use when uploading + */ + makeReferenceName(locale, theme, formFactor, browser) { + const referenceName = [theme, locale, browser, formFactor].filter(function(str) { + if(str !== '' || str != undefined) { return str} + }) + .join("-"); + return referenceName; + } + /** * Zips the latest screenshots. */ @@ -104,7 +120,7 @@ class ScreenshotRequestor { const archiveName = path.join(this.zipFilePath, 'latest.zip'); - // Name the uploaded file reference.zip since the latest screenshots will now be used as the reference screenshots. + // Name the uploaded file the passed referenceName since the latest screenshots will now be used as the reference screenshots. archive.file(archiveName, { name: 'reference.zip' }); await archive.finalize(); @@ -115,12 +131,13 @@ class ScreenshotRequestor { /** * Downloads the screenshots and unzip it to the reference screenshot directory defined by referenceScreenshotsPath. + * @param {string} referenceName - the name of the reference screenshots file to download */ - async downloadScreenshots() { + async downloadScreenshots(referenceName) { let archiveUrl; let fetchOptions; if (this.serviceAuthHeader !== undefined) { - archiveUrl = `${this.serviceUrl}/reference.zip`; + archiveUrl = `${this.serviceUrl}${referenceName}/reference.zip`; fetchOptions = { method: 'GET', headers: { @@ -128,7 +145,7 @@ class ScreenshotRequestor { }, }; } else { - archiveUrl = `${this.url}/reference.zip`; + archiveUrl = `${this.url}${referenceName}/reference.zip`; fetchOptions = { method: 'GET', }; @@ -139,7 +156,7 @@ class ScreenshotRequestor { ); if (response.status === 404) { - logger.info(`No screenshots downloaded from ${this.url}. Either the URL is invalid or no screenshots were previously uploaded.`); + logger.info(`No screenshots downloaded from ${this.url}${referenceName}. Either the URL is invalid or no screenshots were previously uploaded.`); return; } @@ -153,7 +170,7 @@ class ScreenshotRequestor { writeStream.on('finish', async () => { await extract('terra-wdio-screenshots.zip', { dir: this.referenceScreenshotsPath }); fs.removeSync('terra-wdio-screenshots.zip'); - logger.info(`Screenshots downloaded from ${this.url}`); + logger.info(`Screenshots downloaded from ${this.url}${referenceName}`); resolve(); }); } catch (error) { @@ -167,12 +184,13 @@ class ScreenshotRequestor { /** * Uploads the site zip contained in memoryStream * @param {MemoryStream} memoryStream - the MemoryStream to use when uploading + * @param {string} referenceName - the name of the reference zip to use when uploading */ - async uploadScreenshots(memoryStream) { + async uploadScreenshots(memoryStream, referenceName) { const formData = new FormData(); formData.append('file', memoryStream, { filename: 'reference.zip', knownLength: memoryStream.length }); const response = await fetch( - this.serviceUrl, + `${this.serviceUrl}${referenceName}/`, { method: 'PUT', headers: { @@ -183,22 +201,32 @@ class ScreenshotRequestor { ); ScreenshotRequestor.checkStatus(response); - logger.info(`Screenshots are uploaded to ${this.url}`); + logger.info(`Screenshots are uploaded to ${this.url}${referenceName}`); } /** * Downloads the screenshots. + * @param {string} locale - the locale to use when downloading + * @param {string} theme - the theme to use when downloading + * @param {string} formFactor - the formFactor to use when downloading + * @param {string} browser - the browser to use when uploading */ - async download() { - await this.downloadScreenshots(); + async download(locale, theme, formFactor, browser) { + const referenceName = this.makeReferenceName(locale, theme, formFactor, browser); + await this.downloadScreenshots(referenceName); } /** * Uploads the screenshots by deleting the existing screenshots, zipping the new ones, and uploading it + * @param {string} locale - the locale to use when uploading + * @param {string} theme - the theme to use when uploading + * @param {string} formFactor - the formFactor to use when uploading + * @param {string} browser - the browser to use when uploading */ - async upload() { + async upload(locale, theme, formFactor, browser) { + const referenceName = this.makeReferenceName(locale, theme, formFactor, browser); // Delete the existing screenshots from the remote repository because new screenshots will be uploaded. - await this.deleteExistingScreenshots(); + await this.deleteExistingScreenshots(referenceName); // Zip up the existing latest screenshots await this.zipLatestScreenshots(); @@ -207,7 +235,7 @@ class ScreenshotRequestor { const memoryStream = await this.zipDirectoryToMemory(); // Upload the screenshots to the remote repository - await this.uploadScreenshots(memoryStream); + await this.uploadScreenshots(memoryStream, referenceName); // The zipped latest screenshots can now be safely deleted. this.deleteZippedLatestScreenshots(); diff --git a/packages/terra-functional-testing/src/services/wdio-terra-service/WDIOTerraService.js b/packages/terra-functional-testing/src/services/wdio-terra-service/WDIOTerraService.js index 627e23852..d038e61ae 100644 --- a/packages/terra-functional-testing/src/services/wdio-terra-service/WDIOTerraService.js +++ b/packages/terra-functional-testing/src/services/wdio-terra-service/WDIOTerraService.js @@ -64,6 +64,10 @@ class WDIOTerraService { gitToken, issueNumber, useRemoteReferenceScreenshots, + locale, + theme, + formFactor, + browser, } = this.serviceOptions; if (!useRemoteReferenceScreenshots) { @@ -90,7 +94,7 @@ class WDIOTerraService { screenshotConfig = this.getRemoteScreenshotConfiguration(this.screenshotsSites, buildBranch); } const screenshotRequestor = new ScreenshotRequestor(screenshotConfig.publishScreenshotConfiguration); - await screenshotRequestor.download(); + await screenshotRequestor.download(locale, theme, formFactor, browser); } catch (error) { throw new SevereServiceError(error); } @@ -230,12 +234,16 @@ class WDIOTerraService { async uploadBuildBranchScreenshots() { const { buildBranch, + locale, + theme, + formFactor, + browser, } = this.serviceOptions; try { const screenshotConfig = this.getRemoteScreenshotConfiguration(this.screenshotsSites, buildBranch); const screenshotRequestor = new ScreenshotRequestor(screenshotConfig.publishScreenshotConfiguration); - await screenshotRequestor.upload(); + await screenshotRequestor.upload(locale, theme, formFactor, browser); } catch (err) { throw new SevereServiceError(err); } From d7f55d26ddab3498dbc5b5d1eeb5a4ef7fe753d2 Mon Sep 17 00:00:00 2001 From: BenBoersma Date: Mon, 18 Dec 2023 15:10:00 -0600 Subject: [PATCH 02/10] final changes --- .../src/commands/expect/toMatchReference.js | 23 +++++++++++++++---- .../src/commands/validates/screenshot.js | 2 +- .../wdio-terra-service/WDIOTerraService.js | 15 ++++++++---- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/packages/terra-functional-testing/src/commands/expect/toMatchReference.js b/packages/terra-functional-testing/src/commands/expect/toMatchReference.js index a072feaa8..6e361710e 100644 --- a/packages/terra-functional-testing/src/commands/expect/toMatchReference.js +++ b/packages/terra-functional-testing/src/commands/expect/toMatchReference.js @@ -1,4 +1,9 @@ +const path = require('path'); +const fs = require('fs-extra'); +const { Logger } = require('@cerner/terra-cli'); +const getOutputDir = require('../../reporters/spec-reporter/get-output-dir'); const { BUILD_BRANCH, BUILD_TYPE } = require('../../constants'); +const logger = new Logger({ prefix: '[terra-functional-testing:toMatchReference]' }); /** * An assertion method to be paired with Visual Regression Service to assert each screenshot is within * the mismatch tolerance and are the same size. @@ -11,7 +16,7 @@ const { BUILD_BRANCH, BUILD_TYPE } = require('../../constants'); * @param {boolean} screenshot.screenshotWasUpdated - If the reference screenshot was updated with the latest captured screenshot. * @returns {Object} - An object that indicates if the assertion passed or failed with a message. */ -function toMatchReference(screenshot) { +function toMatchReference(screenshot, testName) { const { isNewScreenshot, isSameDimensions, @@ -41,11 +46,19 @@ function toMatchReference(screenshot) { if (global.Terra.serviceOptions.useRemoteReferenceScreenshots && !pass && (global.Terra.serviceOptions.buildBranch.match(BUILD_BRANCH.pullRequest) || global.Terra.serviceOptions.buildType === BUILD_TYPE.branchEventCause)) { pass = true; - process.env.SCREENSHOT_MISMATCH_CHECK = true; - if (message.length > 0) { - message = message.concat('\n'); + const outputDir = getOutputDir(); + const fileName = path.join(outputDir, `ignored-mismatch.json`); + // Create the output directory if it does not already exist. + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + // Since output directory didn't exist file couldnt so just go ahead and make the file + fs.writeFileSync(fileName, JSON.stringify({ screenshotMismatched: true}, null, 2)); + } else if(!fs.existsSync(fileName)) { + // If output directory exists but mismatch file has not been created create one + fs.writeFileSync(fileName, JSON.stringify({ screenshotMismatched: true}, null, 2)); } - message = message.concat('Screenshot has changed and needs to be reviewed.'); + + logger.info(`Test: '${testName}' has a mismatch difference of ${misMatchPercentage}% and needs to be reviewed.`); } return { diff --git a/packages/terra-functional-testing/src/commands/validates/screenshot.js b/packages/terra-functional-testing/src/commands/validates/screenshot.js index 078d366b8..46193bd29 100644 --- a/packages/terra-functional-testing/src/commands/validates/screenshot.js +++ b/packages/terra-functional-testing/src/commands/validates/screenshot.js @@ -22,7 +22,7 @@ const screenshot = (testName, options = {}) => { const screenshotResult = global.browser.checkElement(selector || global.Terra.serviceOptions.selector, wrappedOptions); - global.expect(screenshotResult).toMatchReference(); + global.expect(screenshotResult).toMatchReference(testName); }; module.exports = screenshot; diff --git a/packages/terra-functional-testing/src/services/wdio-terra-service/WDIOTerraService.js b/packages/terra-functional-testing/src/services/wdio-terra-service/WDIOTerraService.js index d038e61ae..a11f0ce2d 100644 --- a/packages/terra-functional-testing/src/services/wdio-terra-service/WDIOTerraService.js +++ b/packages/terra-functional-testing/src/services/wdio-terra-service/WDIOTerraService.js @@ -2,6 +2,7 @@ const path = require('path'); const expect = require('expect'); const fs = require('fs-extra'); const { SevereServiceError } = require('webdriverio'); +const getOutputDir = require('../../reporters/spec-reporter/get-output-dir'); const { accessibility, element, screenshot } = require('../../commands/validates'); const { toBeAccessible, toMatchReference } = require('../../commands/expect'); const { @@ -215,9 +216,7 @@ class WDIOTerraService { ':warning: :bangbang: **WDIO MISMATCH**\n\n', `Check that screenshot change is intended at: ${buildUrl}\n\n`, 'If screenshot change is intended, remote reference screenshots will be updated upon PR merge.\n', - 'If screenshot change is unintended, please fix screenshot issues before PR merge to prevent them from being uploaded.\n\n', - 'Note: This comment only appears the first time a screenshot mismatch is detected on a PR build, ', - 'future builds will need to be checked for unintended screenshot mismatches.', + 'If screenshot change is unintended, please fix screenshot issues before PR merge to prevent them from being uploaded.', ].join(''); const comments = await issue.getComments(); @@ -264,13 +263,21 @@ class WDIOTerraService { return; } + const fileName = path.join(getOutputDir(), `ignored-mismatch.json`); + try { - if (process.env.SCREENSHOT_MISMATCH_CHECK === 'true' && buildBranch.match(BUILD_BRANCH.pullRequest)) { + if (fs.existsSync(fileName) && buildBranch.match(BUILD_BRANCH.pullRequest)) { // We found a screenshot mismatch during our build of this PR branch. await this.postMismatchWarningOnce(); + // Remove mismatch flag file after running + fs.removeSync(fileName); } else if (!buildBranch.match(BUILD_BRANCH.pullRequest) && buildType === BUILD_TYPE.branchEventCause) { // This non-PR branch is being merged or someone pushed code into it directly. await this.uploadBuildBranchScreenshots(); + // Remove mismatch flag file after running if it exists + if(fs.existsSync(fileName)) { + fs.removeSync(fileName); + } } } catch (err) { // The service will stop only if a SevereServiceError is thrown. From 5ab36cd755bd05b24293ef8c0cac7566dc72ade9 Mon Sep 17 00:00:00 2001 From: BenBoersma Date: Mon, 18 Dec 2023 15:14:53 -0600 Subject: [PATCH 03/10] accidently removed logger line, readding it --- .../src/commands/utils/ScreenshotRequestor.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/terra-functional-testing/src/commands/utils/ScreenshotRequestor.js b/packages/terra-functional-testing/src/commands/utils/ScreenshotRequestor.js index bdc5b6ef9..f52514b24 100644 --- a/packages/terra-functional-testing/src/commands/utils/ScreenshotRequestor.js +++ b/packages/terra-functional-testing/src/commands/utils/ScreenshotRequestor.js @@ -70,6 +70,7 @@ class ScreenshotRequestor { // Allow 404s because that just means the screenshots don't exists yet. ScreenshotRequestor.checkStatus(response, [404]); + logger.info('Existing screenshots deleted from remote repository.'); } /** From 30af12474dc58f3789855683a7bc19356d1a7f82 Mon Sep 17 00:00:00 2001 From: BenBoersma Date: Wed, 20 Dec 2023 10:32:16 -0600 Subject: [PATCH 04/10] fixing lint and adding jest tests --- .../src/commands/expect/toMatchReference.js | 11 +- .../src/commands/utils/ScreenshotRequestor.js | 18 +- .../wdio-terra-service/WDIOTerraService.js | 4 +- .../commands/expect/toMatchReference.test.js | 97 ++++- .../utils/ScreenshotRequestor.test.js | 70 +++- .../WDIOTerraService.test.js | 380 ++++++++++++------ 6 files changed, 405 insertions(+), 175 deletions(-) diff --git a/packages/terra-functional-testing/src/commands/expect/toMatchReference.js b/packages/terra-functional-testing/src/commands/expect/toMatchReference.js index 6e361710e..b23d75b98 100644 --- a/packages/terra-functional-testing/src/commands/expect/toMatchReference.js +++ b/packages/terra-functional-testing/src/commands/expect/toMatchReference.js @@ -3,6 +3,7 @@ const fs = require('fs-extra'); const { Logger } = require('@cerner/terra-cli'); const getOutputDir = require('../../reporters/spec-reporter/get-output-dir'); const { BUILD_BRANCH, BUILD_TYPE } = require('../../constants'); + const logger = new Logger({ prefix: '[terra-functional-testing:toMatchReference]' }); /** * An assertion method to be paired with Visual Regression Service to assert each screenshot is within @@ -47,17 +48,17 @@ function toMatchReference(screenshot, testName) { && (global.Terra.serviceOptions.buildBranch.match(BUILD_BRANCH.pullRequest) || global.Terra.serviceOptions.buildType === BUILD_TYPE.branchEventCause)) { pass = true; const outputDir = getOutputDir(); - const fileName = path.join(outputDir, `ignored-mismatch.json`); + const fileName = path.join(outputDir, 'ignored-mismatch.json'); // Create the output directory if it does not already exist. if (!fs.existsSync(outputDir)) { fs.mkdirSync(outputDir, { recursive: true }); // Since output directory didn't exist file couldnt so just go ahead and make the file - fs.writeFileSync(fileName, JSON.stringify({ screenshotMismatched: true}, null, 2)); - } else if(!fs.existsSync(fileName)) { + fs.writeFileSync(fileName, JSON.stringify({ screenshotMismatched: true }, null, 2)); + } else if (!fs.existsSync(fileName)) { // If output directory exists but mismatch file has not been created create one - fs.writeFileSync(fileName, JSON.stringify({ screenshotMismatched: true}, null, 2)); + fs.writeFileSync(fileName, JSON.stringify({ screenshotMismatched: true }, null, 2)); } - + logger.info(`Test: '${testName}' has a mismatch difference of ${misMatchPercentage}% and needs to be reviewed.`); } diff --git a/packages/terra-functional-testing/src/commands/utils/ScreenshotRequestor.js b/packages/terra-functional-testing/src/commands/utils/ScreenshotRequestor.js index f52514b24..109ebbc3d 100644 --- a/packages/terra-functional-testing/src/commands/utils/ScreenshotRequestor.js +++ b/packages/terra-functional-testing/src/commands/utils/ScreenshotRequestor.js @@ -89,11 +89,15 @@ class ScreenshotRequestor { * @param {string} formFactor - the formFactor to use when downloading * @param {string} browser - the browser to use when uploading */ - makeReferenceName(locale, theme, formFactor, browser) { - const referenceName = [theme, locale, browser, formFactor].filter(function(str) { - if(str !== '' || str != undefined) { return str} - }) - .join("-"); + static makeReferenceName(locale, theme, formFactor, browser) { + const referenceName = [theme, locale, browser, formFactor].filter( + (str) => { + if (str !== '' || str !== undefined) { + return str; + } + return false; + }, + ).join('-'); return referenceName; } @@ -213,7 +217,7 @@ class ScreenshotRequestor { * @param {string} browser - the browser to use when uploading */ async download(locale, theme, formFactor, browser) { - const referenceName = this.makeReferenceName(locale, theme, formFactor, browser); + const referenceName = ScreenshotRequestor.makeReferenceName(locale, theme, formFactor, browser); await this.downloadScreenshots(referenceName); } @@ -225,7 +229,7 @@ class ScreenshotRequestor { * @param {string} browser - the browser to use when uploading */ async upload(locale, theme, formFactor, browser) { - const referenceName = this.makeReferenceName(locale, theme, formFactor, browser); + const referenceName = ScreenshotRequestor.makeReferenceName(locale, theme, formFactor, browser); // Delete the existing screenshots from the remote repository because new screenshots will be uploaded. await this.deleteExistingScreenshots(referenceName); diff --git a/packages/terra-functional-testing/src/services/wdio-terra-service/WDIOTerraService.js b/packages/terra-functional-testing/src/services/wdio-terra-service/WDIOTerraService.js index a11f0ce2d..135a0409e 100644 --- a/packages/terra-functional-testing/src/services/wdio-terra-service/WDIOTerraService.js +++ b/packages/terra-functional-testing/src/services/wdio-terra-service/WDIOTerraService.js @@ -263,7 +263,7 @@ class WDIOTerraService { return; } - const fileName = path.join(getOutputDir(), `ignored-mismatch.json`); + const fileName = path.join(getOutputDir(), 'ignored-mismatch.json'); try { if (fs.existsSync(fileName) && buildBranch.match(BUILD_BRANCH.pullRequest)) { @@ -275,7 +275,7 @@ class WDIOTerraService { // This non-PR branch is being merged or someone pushed code into it directly. await this.uploadBuildBranchScreenshots(); // Remove mismatch flag file after running if it exists - if(fs.existsSync(fileName)) { + if (fs.existsSync(fileName)) { fs.removeSync(fileName); } } diff --git a/packages/terra-functional-testing/tests/jest/commands/expect/toMatchReference.test.js b/packages/terra-functional-testing/tests/jest/commands/expect/toMatchReference.test.js index aa7e6883f..9d1ade389 100644 --- a/packages/terra-functional-testing/tests/jest/commands/expect/toMatchReference.test.js +++ b/packages/terra-functional-testing/tests/jest/commands/expect/toMatchReference.test.js @@ -1,8 +1,22 @@ +const path = require('path'); +const fs = require('fs-extra'); +const mockInfo = jest.fn(); +jest.mock('@cerner/terra-cli/lib/utils/Logger', () => function mock() { + return { + info: mockInfo, + }; +}); +jest.mock('../../../../src/reporters/spec-reporter/get-output-dir', () => ( + jest.fn().mockImplementation(() => ('/mock/')) +)); + const toMatchReference = require('../../../../src/commands/expect/toMatchReference'); const { BUILD_BRANCH, BUILD_TYPE } = require('../../../../src/constants'); describe('toMatchReference', () => { beforeAll(() => { + mockInfo.mockClear(); + jest.resetModules(); global.Terra = { serviceOptions: { ignoreScreenshotMismatch: false, @@ -10,6 +24,11 @@ describe('toMatchReference', () => { }; }); + afterEach(() => { + // Restore all fs mocks. + jest.restoreAllMocks(); + }); + it('should pass if matches reference screenshot', () => { const receivedScreenshot = { isNewScreenshot: false, @@ -240,12 +259,16 @@ describe('toMatchReference', () => { isSameDimensions: false, misMatchPercentage: 0.10, }; + jest.spyOn(fs, 'existsSync').mockImplementation(() => true); + jest.spyOn(fs, 'existsSync').mockImplementation(() => true); - const result = toMatchReference(receivedScreenshot); - const expectedMessage = 'Expected the screenshot to match the reference screenshot, but received a screenshot with different dimensions.\nExpected the screenshot to be within the mismatch tolerance, but received a mismatch difference of 0.1%.\nScreenshot has changed and needs to be reviewed.'; + const result = toMatchReference(receivedScreenshot, 'TestName'); + const expectedMessage = 'Expected the screenshot to match the reference screenshot, but received a screenshot with different dimensions.\nExpected the screenshot to be within the mismatch tolerance, but received a mismatch difference of 0.1%.'; expect(result.pass).toBe(true); + expect(fs.existsSync).toHaveBeenCalledTimes(2); expect(result.message()).toEqual(expectedMessage); + expect(mockInfo).toHaveBeenCalledWith('Test: \'TestName\' has a mismatch difference of 0.1% and needs to be reviewed.'); }); it('should not pass if not within mismatch tolerance, buildBranch matches master, useRemoteReferenceScreenshots is true and buildType is not defined', () => { @@ -279,12 +302,15 @@ describe('toMatchReference', () => { isSameDimensions: false, misMatchPercentage: 0.10, }; + jest.spyOn(fs, 'existsSync').mockImplementation(() => true); - const result = toMatchReference(receivedScreenshot); - const expectedMessage = 'Expected the screenshot to match the reference screenshot, but received a screenshot with different dimensions.\nExpected the screenshot to be within the mismatch tolerance, but received a mismatch difference of 0.1%.\nScreenshot has changed and needs to be reviewed.'; + const result = toMatchReference(receivedScreenshot, 'TestName'); + const expectedMessage = 'Expected the screenshot to match the reference screenshot, but received a screenshot with different dimensions.\nExpected the screenshot to be within the mismatch tolerance, but received a mismatch difference of 0.1%.'; expect(result.pass).toBe(true); + expect(fs.existsSync).toHaveBeenCalledTimes(2); expect(result.message()).toEqual(expectedMessage); + expect(mockInfo).toHaveBeenCalledWith('Test: \'TestName\' has a mismatch difference of 0.1% and needs to be reviewed.'); }); it('should pass if not within mismatch tolerance but buildBranch matches master, useRemoteReferenceScreenshots is true, and buildType is branchEventCause', () => { @@ -299,12 +325,69 @@ describe('toMatchReference', () => { isSameDimensions: false, misMatchPercentage: 0.10, }; + jest.spyOn(fs, 'existsSync').mockImplementation(() => true); - const result = toMatchReference(receivedScreenshot); - const expectedMessage = 'Expected the screenshot to match the reference screenshot, but received a screenshot with different dimensions.\nExpected the screenshot to be within the mismatch tolerance, but received a mismatch difference of 0.1%.\nScreenshot has changed and needs to be reviewed.'; + const result = toMatchReference(receivedScreenshot, 'TestName'); + const expectedMessage = 'Expected the screenshot to match the reference screenshot, but received a screenshot with different dimensions.\nExpected the screenshot to be within the mismatch tolerance, but received a mismatch difference of 0.1%.'; + + expect(result.pass).toBe(true); + expect(result.message()).toEqual(expectedMessage); + expect(mockInfo).toHaveBeenCalledWith('Test: \'TestName\' has a mismatch difference of 0.1% and needs to be reviewed.'); + }); + it('should create the output directory and ignored-mismatch file if useRemoteReferenceScreenshots is true, a mismatch happened, and it is a pull request', () => { + global.Terra = { + serviceOptions: { + buildBranch: BUILD_BRANCH.master, + buildType: BUILD_TYPE.branchEventCause, + useRemoteReferenceScreenshots: true, + }, + }; + const receivedScreenshot = { + isSameDimensions: false, + misMatchPercentage: 0.10, + }; + jest.spyOn(fs, 'existsSync').mockImplementationOnce(() => false); + jest.spyOn(fs, 'mkdirSync').mockImplementationOnce(() => false); + jest.spyOn(fs, 'writeFileSync').mockImplementationOnce(() => false); + + const result = toMatchReference(receivedScreenshot, 'TestName'); + const expectedMessage = 'Expected the screenshot to match the reference screenshot, but received a screenshot with different dimensions.\nExpected the screenshot to be within the mismatch tolerance, but received a mismatch difference of 0.1%.'; + + expect(result.pass).toBe(true); + expect(fs.existsSync).toHaveBeenCalledTimes(1); + expect(fs.mkdirSync).toHaveBeenCalledWith('/mock/', { recursive: true }); + expect(fs.writeFileSync).toHaveBeenCalledWith(path.join('/mock/', 'ignored-mismatch.json'), JSON.stringify({ screenshotMismatched: true }, null, 2)); + expect(result.message()).toEqual(expectedMessage); + expect(mockInfo).toHaveBeenCalledWith('Test: \'TestName\' has a mismatch difference of 0.1% and needs to be reviewed.'); + }); + + it('should just create the ignored-mismatch file if output directory already exists but file does not', () => { + global.Terra = { + serviceOptions: { + buildBranch: BUILD_BRANCH.master, + buildType: BUILD_TYPE.branchEventCause, + useRemoteReferenceScreenshots: true, + }, + }; + const receivedScreenshot = { + isSameDimensions: false, + misMatchPercentage: 0.10, + }; + jest.spyOn(fs, 'existsSync').mockImplementation((pathName) => (pathName === '/mock/') ? true : false); + jest.spyOn(fs, 'writeFileSync').mockImplementationOnce(() => false); + + const result = toMatchReference(receivedScreenshot, 'TestName'); + const expectedMessage = 'Expected the screenshot to match the reference screenshot, but received a screenshot with different dimensions.\nExpected the screenshot to be within the mismatch tolerance, but received a mismatch difference of 0.1%.'; + expect(result.pass).toBe(true); + expect(fs.existsSync).toHaveBeenCalledTimes(2); + expect(fs.existsSync.mock.calls).toEqual([ + ['/mock/'], // First call + [path.join('/mock/', 'ignored-mismatch.json')] // Second call + ]) + expect(fs.writeFileSync).toHaveBeenCalledWith(path.join('/mock/', 'ignored-mismatch.json'), JSON.stringify({ screenshotMismatched: true }, null, 2)); expect(result.message()).toEqual(expectedMessage); - expect(process.env.SCREENSHOT_MISMATCH_CHECK).toBe('true'); + expect(mockInfo).toHaveBeenCalledWith('Test: \'TestName\' has a mismatch difference of 0.1% and needs to be reviewed.'); }); }); diff --git a/packages/terra-functional-testing/tests/jest/commands/utils/ScreenshotRequestor.test.js b/packages/terra-functional-testing/tests/jest/commands/utils/ScreenshotRequestor.test.js index 3f129cd41..172c02db7 100644 --- a/packages/terra-functional-testing/tests/jest/commands/utils/ScreenshotRequestor.test.js +++ b/packages/terra-functional-testing/tests/jest/commands/utils/ScreenshotRequestor.test.js @@ -99,10 +99,10 @@ describe('ScreenshotRequestor', () => { status: 200, }); - await screenshotRequestor.deleteExistingScreenshots(); + await screenshotRequestor.deleteExistingScreenshots('mock'); expect(fetch).toHaveBeenCalledWith( - screenshotRequestor.serviceUrl, + `${screenshotRequestor.serviceUrl}mock/`, { method: 'DELETE', headers: { @@ -181,9 +181,9 @@ describe('ScreenshotRequestor', () => { zipFilePath: path.join(process.cwd(), 'zip-path'), }); - await expect(screenshotRequestor.downloadScreenshots()).resolves.toBe(); + await expect(screenshotRequestor.downloadScreenshots('mock')).resolves.toBe(); expect(fetch).toHaveBeenCalledWith( - `${screenshotRequestor.serviceUrl}/reference.zip`, + `${screenshotRequestor.serviceUrl}mock/reference.zip`, { method: 'GET', headers: { @@ -198,7 +198,7 @@ describe('ScreenshotRequestor', () => { expect(mockOnFinish).toBeCalledWith('finish', expect.any(Function)); expect(extract).toHaveBeenCalledWith('terra-wdio-screenshots.zip', { dir: screenshotRequestor.referenceScreenshotsPath }); expect(fs.removeSync).toHaveBeenCalledWith('terra-wdio-screenshots.zip'); - expect(mockInfo).toHaveBeenCalledWith(`Screenshots downloaded from ${screenshotRequestor.url}`); + expect(mockInfo).toHaveBeenCalledWith(`Screenshots downloaded from ${screenshotRequestor.url}mock`); }); it('calls out an errors', async () => { @@ -223,9 +223,9 @@ describe('ScreenshotRequestor', () => { zipFilePath: path.join(process.cwd(), 'zip-path'), }); - await expect(screenshotRequestor.downloadScreenshots()).rejects.toBe(); + await expect(screenshotRequestor.downloadScreenshots('mock')).rejects.toBe(); expect(fetch).toHaveBeenCalledWith( - `${screenshotRequestor.serviceUrl}/reference.zip`, + `${screenshotRequestor.serviceUrl}mock/reference.zip`, { method: 'GET', headers: { @@ -255,9 +255,9 @@ describe('ScreenshotRequestor', () => { zipFilePath: path.join(process.cwd(), 'zip-path'), }); - await screenshotRequestor.downloadScreenshots(); + await screenshotRequestor.downloadScreenshots('mock'); expect(fetch).toHaveBeenCalledWith( - `${screenshotRequestor.serviceUrl}/reference.zip`, + `${screenshotRequestor.serviceUrl}mock/reference.zip`, { method: 'GET', headers: { @@ -265,7 +265,7 @@ describe('ScreenshotRequestor', () => { }, }, ); - expect(mockInfo).toHaveBeenCalledWith(`No screenshots downloaded from ${screenshotRequestor.url}. Either the URL is invalid or no screenshots were previously uploaded.`); + expect(mockInfo).toHaveBeenCalledWith(`No screenshots downloaded from ${screenshotRequestor.url}mock. Either the URL is invalid or no screenshots were previously uploaded.`); }); it('will call fetch with the public url if serviceAuthHeader is undefined', async () => { @@ -283,14 +283,14 @@ describe('ScreenshotRequestor', () => { zipFilePath: path.join(process.cwd(), 'zip-path'), }); - await screenshotRequestor.downloadScreenshots(); + await screenshotRequestor.downloadScreenshots('mock'); expect(fetch).toHaveBeenCalledWith( - `${screenshotRequestor.url}/reference.zip`, + `${screenshotRequestor.url}mock/reference.zip`, { method: 'GET', }, ); - expect(mockInfo).toHaveBeenCalledWith(`No screenshots downloaded from ${screenshotRequestor.url}. Either the URL is invalid or no screenshots were previously uploaded.`); + expect(mockInfo).toHaveBeenCalledWith(`No screenshots downloaded from ${screenshotRequestor.url}mock. Either the URL is invalid or no screenshots were previously uploaded.`); }); }); @@ -315,13 +315,13 @@ describe('ScreenshotRequestor', () => { const memoryStream = { length: 1000, }; - await screenshotRequestor.uploadScreenshots(memoryStream); + await screenshotRequestor.uploadScreenshots(memoryStream, 'mock'); expect(FormData).toHaveBeenCalled(); const mockFormDataInstance = FormData.mock.instances[0]; expect(mockFormDataInstance.append).toHaveBeenCalledWith('file', memoryStream, { filename: 'reference.zip', knownLength: 1000 }); expect(fetch).toHaveBeenCalledWith( - screenshotRequestor.serviceUrl, + `${screenshotRequestor.serviceUrl}mock/`, { method: 'PUT', headers: { @@ -332,34 +332,41 @@ describe('ScreenshotRequestor', () => { ); expect(ScreenshotRequestor.checkStatus).toHaveBeenCalled(); ScreenshotRequestor.checkStatus = oldCheckStatus; - expect(mockInfo).toHaveBeenCalledWith(`Screenshots are uploaded to ${screenshotRequestor.url}`); + expect(mockInfo).toHaveBeenCalledWith(`Screenshots are uploaded to ${screenshotRequestor.url}mock`); }); }); describe('download', () => { it('calls downloadScreenshots', async () => { const oldDownloadScreenshots = ScreenshotRequestor.prototype.downloadScreenshots; + const oldMakeReferenceName = ScreenshotRequestor.makeReferenceName; ScreenshotRequestor.prototype.downloadScreenshots = jest.fn(); + ScreenshotRequestor.makeReferenceName = jest.fn(); const screenshotRequestor = new ScreenshotRequestor({}); + const referenceName = 'locale-theme-formFactor-browser'; + ScreenshotRequestor.makeReferenceName.mockImplementationOnce(() => referenceName); screenshotRequestor.downloadScreenshots.mockResolvedValueOnce(); - await screenshotRequestor.download(); + await screenshotRequestor.download('locale', 'theme', 'formFactor', 'browser'); - expect(screenshotRequestor.downloadScreenshots).toHaveBeenCalled(); + expect(ScreenshotRequestor.makeReferenceName).toHaveBeenCalledWith('locale', 'theme', 'formFactor', 'browser'); + expect(screenshotRequestor.downloadScreenshots).toHaveBeenCalledWith(referenceName); ScreenshotRequestor.prototype.downloadScreenshots = oldDownloadScreenshots; + ScreenshotRequestor.makeReferenceName = oldMakeReferenceName; }); }); describe('upload', () => { it('deletes the existing screenshots and zips and uploads the new screenshots', async () => { + const oldMakeReferenceName = ScreenshotRequestor.makeReferenceName; const oldDeleteExistingScreenshots = ScreenshotRequestor.prototype.deleteExistingScreenshots; const oldZipLatestScreenshots = ScreenshotRequestor.prototype.zipLatestScreenshots; const oldzipDirectoryToMemory = ScreenshotRequestor.prototype.zipDirectoryToMemory; const oldUploadScreenshots = ScreenshotRequestor.prototype.uploadScreenshots; const oldDeleteZippedLatestScreenshots = ScreenshotRequestor.prototype.deleteZippedLatestScreenshots; - ScreenshotRequestor.checkStatus = jest.fn(); + ScreenshotRequestor.makeReferenceName = jest.fn(); ScreenshotRequestor.prototype.deleteExistingScreenshots = jest.fn(); ScreenshotRequestor.prototype.zipLatestScreenshots = jest.fn(); ScreenshotRequestor.prototype.zipDirectoryToMemory = jest.fn(); @@ -377,17 +384,22 @@ describe('ScreenshotRequestor', () => { const memoryStream = { length: 1000, }; + const referenceName = 'locale-theme-formFactor-browser'; + ScreenshotRequestor.makeReferenceName.mockImplementationOnce(() => referenceName); screenshotRequestor.deleteExistingScreenshots.mockResolvedValueOnce(); screenshotRequestor.zipLatestScreenshots.mockResolvedValueOnce(); screenshotRequestor.zipDirectoryToMemory.mockResolvedValueOnce(memoryStream); screenshotRequestor.uploadScreenshots.mockResolvedValueOnce(); screenshotRequestor.deleteZippedLatestScreenshots.mockResolvedValueOnce(); - await screenshotRequestor.upload(); + await screenshotRequestor.upload('locale', 'theme', 'formFactor', 'browser'); - expect(screenshotRequestor.deleteExistingScreenshots).toHaveBeenCalled(); + expect(ScreenshotRequestor.makeReferenceName).toHaveBeenCalledWith('locale', 'theme', 'formFactor', 'browser'); + expect(screenshotRequestor.deleteExistingScreenshots).toHaveBeenCalledWith(referenceName); expect(screenshotRequestor.zipDirectoryToMemory).toHaveBeenCalled(); - expect(screenshotRequestor.uploadScreenshots).toHaveBeenCalledWith(memoryStream); + expect(screenshotRequestor.uploadScreenshots).toHaveBeenCalledWith(memoryStream, referenceName) ; + expect(screenshotRequestor.deleteZippedLatestScreenshots).toHaveBeenCalled() ; + ScreenshotRequestor.makeReferenceName = oldMakeReferenceName; ScreenshotRequestor.prototype.deleteExistingScreenshots = oldDeleteExistingScreenshots; ScreenshotRequestor.prototype.zipLatestScreenshots = oldZipLatestScreenshots; ScreenshotRequestor.prototype.zipDirectoryToMemory = oldzipDirectoryToMemory; @@ -395,4 +407,18 @@ describe('ScreenshotRequestor', () => { ScreenshotRequestor.prototype.deleteZippedLatestScreenshots = oldDeleteZippedLatestScreenshots; }); }); + + describe('makeReferenceName', () => { + it('should return a joined string of arguments', () => { + expect(ScreenshotRequestor.makeReferenceName('locale', 'theme', 'formFactor', 'browser')).toEqual('theme-locale-browser-formFactor'); + }); + + it('should should skip undefined arguments', () => { + expect(ScreenshotRequestor.makeReferenceName('locale', undefined, undefined, 'browser')).toEqual('locale-browser'); + }); + + it('should return empty if no arguments are given', () => { + expect(ScreenshotRequestor.makeReferenceName()).toEqual(''); + }); + }); }); diff --git a/packages/terra-functional-testing/tests/jest/services/wdio-terra-service/WDIOTerraService.test.js b/packages/terra-functional-testing/tests/jest/services/wdio-terra-service/WDIOTerraService.test.js index d775d2a94..3546bf39c 100644 --- a/packages/terra-functional-testing/tests/jest/services/wdio-terra-service/WDIOTerraService.test.js +++ b/packages/terra-functional-testing/tests/jest/services/wdio-terra-service/WDIOTerraService.test.js @@ -16,6 +16,9 @@ fs.readJson.mockResolvedValue({ url: `git+https://github.com/${repoOwner}/${repoName}.git`, }, }); +jest.mock('../../../../src/reporters/spec-reporter/get-output-dir', () => ( + jest.fn().mockImplementation(() => ('/mock/')) +)); describe('WDIO Terra Service', () => { describe('getRepoMetadata', () => { @@ -239,17 +242,23 @@ describe('WDIO Terra Service', () => { describe('onComplete hook', () => { let config; let getRemoteScreenshotConfiguration; + let buildUrl; beforeAll(() => { getRemoteScreenshotConfiguration = jest.fn(() => ({ publishScreenshotConfiguration: jest.fn(), })); + buildUrl = 'https://example.com/buildUrl'; }); beforeEach(() => { config = { getRemoteScreenshotConfiguration, screenshotsSites: 'screenshot sites object', + serviceOptions: { + useRemoteReferenceScreenshots: true, + buildUrl, + }, }; }); @@ -261,159 +270,266 @@ describe('WDIO Terra Service', () => { jest.restoreAllMocks(); }); - describe('Posting a mismatch warning comment to the github issue', () => { - let postComment; - let getComments; - let buildUrl; - let warningMessage; - - beforeAll(() => { - postComment = jest.spyOn(GithubIssue.prototype, 'postComment'); - getComments = jest.spyOn(GithubIssue.prototype, 'getComments'); - buildUrl = 'https://example.com/buildUrl'; - warningMessage = [ - ':warning: :bangbang: **WDIO MISMATCH**\n\n', - `Check that screenshot change is intended at: ${buildUrl}\n\n`, - 'If screenshot change is intended, remote reference screenshots will be updated upon PR merge.\n', - 'If screenshot change is unintended, please fix screenshot issues before PR merge to prevent them from being uploaded.\n\n', - 'Note: This comment only appears the first time a screenshot mismatch is detected on a PR build, ', - 'future builds will need to be checked for unintended screenshot mismatches.', - ].join(''); - }); + it('should call postMismatchWarningOnce if ignored-mismatch file exists and branch is pull-request', async () => { + const oldPostMismatchWarningOnce = WDIOTerraService.prototype.postMismatchWarningOnce; + WDIOTerraService.prototype.postMismatchWarningOnce = jest.fn(); + jest.spyOn(fs, 'existsSync').mockImplementation(() => true); + jest.spyOn(fs, 'removeSync').mockImplementation(); + config.serviceOptions.buildBranch = 'pr-123'; + const service = new WDIOTerraService({}, {}, config); + service.postMismatchWarningOnce.mockResolvedValueOnce(); - beforeEach(() => { - // Happy path environs and config. - process.env.SCREENSHOT_MISMATCH_CHECK = true; - config.serviceOptions = { - useRemoteReferenceScreenshots: true, - buildBranch: 'pr-123', - buildUrl, - }; - }); + await service.onComplete(); - afterAll(() => { - process.env.SCREENSHOT_MISMATCH_CHECK = undefined; - }); + expect(fs.existsSync).toHaveBeenCalledWith('/mock/ignored-mismatch.json'); + expect(service.postMismatchWarningOnce).toHaveBeenCalled(); + expect(fs.removeSync).toHaveBeenCalledWith('/mock/ignored-mismatch.json'); - it('Posts a comment', async () => { - getComments.mockResolvedValue(['not the warning message']); - postComment.mockResolvedValue(); - const service = new WDIOTerraService({}, {}, config); - await service.onComplete(); - expect(postComment).toHaveBeenCalledWith(warningMessage); - }); + WDIOTerraService.prototype.postMismatchWarningOnce = oldPostMismatchWarningOnce; + config.serviceOptions.buildBranch = undefined; + }); - it('Does not try to post if no mismatch has been found', async () => { - process.env.SCREENSHOT_MISMATCH_CHECK = undefined; - getComments.mockResolvedValue(['not the warning message']); - postComment.mockResolvedValue(); - const service = new WDIOTerraService({}, {}, config); - await service.onComplete(); - expect(postComment).not.toHaveBeenCalled(); - }); + it('should not call postMismatchWarningOnce if ignored-mismatch file does not exist', async () => { + const oldPostMismatchWarningOnce = WDIOTerraService.prototype.postMismatchWarningOnce; + WDIOTerraService.prototype.postMismatchWarningOnce = jest.fn(); + jest.spyOn(fs, 'existsSync').mockImplementation(() => false); + jest.spyOn(fs, 'removeSync').mockImplementation(); + config.serviceOptions.buildBranch = 'pr-123'; + const service = new WDIOTerraService({}, {}, config); + service.postMismatchWarningOnce.mockResolvedValueOnce(); - it('Does not try to post if we are not using remote screenshots', async () => { - config.serviceOptions.useRemoteReferenceScreenshots = false; - getComments.mockResolvedValue(['not the warning message']); - postComment.mockResolvedValue(); - const service = new WDIOTerraService({}, {}, config); - await service.onComplete(); - expect(postComment).not.toHaveBeenCalled(); - }); + await service.onComplete(); - it('Does not try to post if the branch does not match the PR pattern', async () => { - config.serviceOptions.buildBranch = 'not-a-pr'; - getComments.mockResolvedValue(['not the warning message']); - postComment.mockResolvedValue(); - const service = new WDIOTerraService({}, {}, config); - await service.onComplete(); - expect(postComment).not.toHaveBeenCalled(); - }); + expect(fs.existsSync).toHaveBeenCalledWith('/mock/ignored-mismatch.json'); + expect(service.postMismatchWarningOnce).not.toHaveBeenCalled(); + expect(fs.removeSync).not.toHaveBeenCalled(); - it('Does not post if the comment is found on the issue', async () => { - getComments.mockResolvedValue([warningMessage]); - postComment.mockResolvedValue(); - const service = new WDIOTerraService({}, {}, config); - await service.onComplete(); - expect(getComments).toHaveBeenCalled(); - expect(postComment).not.toHaveBeenCalled(); - }); + WDIOTerraService.prototype.postMismatchWarningOnce = oldPostMismatchWarningOnce; + config.serviceOptions.buildBranch = undefined; + }); - it('Stops the service without posting if it fails to get the issue comments', async () => { - getComments.mockRejectedValue('oh no!'); - const service = new WDIOTerraService({}, {}, config); - await expect(service.onComplete()).rejects.toThrow(SevereServiceError); - expect(getComments).toHaveBeenCalled(); - expect(postComment).not.toHaveBeenCalled(); - }); + it('should not call postMismatchWarningOnce if branch is not pull-request', async () => { + const oldPostMismatchWarningOnce = WDIOTerraService.prototype.postMismatchWarningOnce; + WDIOTerraService.prototype.postMismatchWarningOnce = jest.fn(); + jest.spyOn(fs, 'existsSync').mockImplementation(() => true); + jest.spyOn(fs, 'removeSync').mockImplementation(); + config.serviceOptions.buildBranch = 'master'; + const service = new WDIOTerraService({}, {}, config); + service.postMismatchWarningOnce.mockResolvedValueOnce(); - it('Stops the service if it fails to post the comment', async () => { - getComments.mockResolvedValue(['not the warning message']); - postComment.mockRejectedValue('oh no!'); - const service = new WDIOTerraService({}, {}, config); - await expect(service.onComplete()).rejects.toThrow(SevereServiceError); - expect(postComment).toHaveBeenCalled(); - }); + await service.onComplete(); + + expect(fs.existsSync).toHaveBeenCalledWith('/mock/ignored-mismatch.json'); + expect(service.postMismatchWarningOnce).not.toHaveBeenCalled(); + expect(fs.removeSync).not.toHaveBeenCalled(); + + WDIOTerraService.prototype.postMismatchWarningOnce = oldPostMismatchWarningOnce; + config.serviceOptions.buildBranch = undefined; }); - describe('Uploading screenshots to a remote repo', () => { - let upload; - const buildBranch = 'not-a-pull-request'; + it('should call uploadBuildBranchScreenshots if build type is a commit and it is not a pull request', async () => { + const oldUploadBuildBranchScreenshots = WDIOTerraService.prototype.uploadBuildBranchScreenshots; + WDIOTerraService.prototype.uploadBuildBranchScreenshots = jest.fn(); + jest.spyOn(fs, 'existsSync').mockImplementation(() => false); + jest.spyOn(fs, 'removeSync').mockImplementation(); + config.serviceOptions.buildBranch = 'master'; + config.serviceOptions.buildType = BUILD_TYPE.branchEventCause; + const service = new WDIOTerraService({}, {}, config); + service.uploadBuildBranchScreenshots.mockResolvedValueOnce(); - beforeAll(() => { - upload = jest.spyOn(ScreenshotRequestor.prototype, 'upload'); - }); + await service.onComplete(); + + expect(fs.existsSync).toHaveBeenCalledWith('/mock/ignored-mismatch.json'); + expect(service.uploadBuildBranchScreenshots).toHaveBeenCalled(); + expect(fs.removeSync).not.toHaveBeenCalled(); - beforeEach(() => { - config.serviceOptions = { + WDIOTerraService.prototype.uploadBuildBranchScreenshots = oldUploadBuildBranchScreenshots; + config.serviceOptions.buildBranch = undefined; + }); + + it('should call uploadBuildBranchScreenshots and remove the ignored-mismatch file if it exists', async () => { + const oldUploadBuildBranchScreenshots = WDIOTerraService.prototype.uploadBuildBranchScreenshots; + WDIOTerraService.prototype.uploadBuildBranchScreenshots = jest.fn(); + jest.spyOn(fs, 'existsSync').mockImplementation(() => true); + jest.spyOn(fs, 'removeSync').mockImplementation(); + config.serviceOptions.buildBranch = 'master'; + config.serviceOptions.buildType = BUILD_TYPE.branchEventCause; + const service = new WDIOTerraService({}, {}, config); + service.uploadBuildBranchScreenshots.mockResolvedValueOnce(); + + await service.onComplete(); + + expect(fs.existsSync).toHaveBeenCalledWith('/mock/ignored-mismatch.json'); + expect(service.uploadBuildBranchScreenshots).toHaveBeenCalled(); + expect(fs.removeSync).toHaveBeenCalledWith('/mock/ignored-mismatch.json'); + + WDIOTerraService.prototype.uploadBuildBranchScreenshots = oldUploadBuildBranchScreenshots; + config.serviceOptions.buildBranch = undefined; + }); + + it('should not call uploadBuildBranchScreenshots if build is a PR', async () => { + const oldUploadBuildBranchScreenshots = WDIOTerraService.prototype.uploadBuildBranchScreenshots; + WDIOTerraService.prototype.uploadBuildBranchScreenshots = jest.fn(); + jest.spyOn(fs, 'existsSync').mockImplementation(() => false); + jest.spyOn(fs, 'removeSync').mockImplementation(); + config.serviceOptions.buildBranch = 'pr-123'; + config.serviceOptions.buildType = BUILD_TYPE.branchEventCause; + const service = new WDIOTerraService({}, {}, config); + service.uploadBuildBranchScreenshots.mockResolvedValueOnce(); + + await service.onComplete(); + + expect(fs.existsSync).toHaveBeenCalledWith('/mock/ignored-mismatch.json'); + expect(service.uploadBuildBranchScreenshots).not.toHaveBeenCalled(); + expect(fs.removeSync).not.toHaveBeenCalled(); + + WDIOTerraService.prototype.uploadBuildBranchScreenshots = oldUploadBuildBranchScreenshots; + config.serviceOptions.buildBranch = undefined; + }); + + it('should not call uploadBuildBranchScreenshots if build type is not a branchEventCause', async () => { + const oldUploadBuildBranchScreenshots = WDIOTerraService.prototype.uploadBuildBranchScreenshots; + WDIOTerraService.prototype.uploadBuildBranchScreenshots = jest.fn(); + jest.spyOn(fs, 'existsSync').mockImplementation(() => false); + jest.spyOn(fs, 'removeSync').mockImplementation(); + config.serviceOptions.buildBranch = 'master'; + config.serviceOptions.buildType = 'Replayed'; + const service = new WDIOTerraService({}, {}, config); + service.uploadBuildBranchScreenshots.mockResolvedValueOnce(); + + await service.onComplete(); + + expect(fs.existsSync).toHaveBeenCalledWith('/mock/ignored-mismatch.json'); + expect(service.uploadBuildBranchScreenshots).not.toHaveBeenCalled(); + expect(fs.removeSync).not.toHaveBeenCalled(); + + WDIOTerraService.prototype.uploadBuildBranchScreenshots = oldUploadBuildBranchScreenshots; + config.serviceOptions.buildBranch = undefined; + }); + }); + + describe('postMismatchWarningOnce function', () => { + let postComment; + let getComments; + let buildUrl; + let warningMessage; + let config; + + beforeAll(() => { + postComment = jest.spyOn(GithubIssue.prototype, 'postComment'); + getComments = jest.spyOn(GithubIssue.prototype, 'getComments'); + buildUrl = 'https://example.com/buildUrl'; + warningMessage = [ + ':warning: :bangbang: **WDIO MISMATCH**\n\n', + `Check that screenshot change is intended at: ${buildUrl}\n\n`, + 'If screenshot change is intended, remote reference screenshots will be updated upon PR merge.\n', + 'If screenshot change is unintended, please fix screenshot issues before PR merge to prevent them from being uploaded.', + ].join(''); + }); + + beforeEach(() => { + config = { + screenshotsSites: 'screenshot sites object', + serviceOptions: { useRemoteReferenceScreenshots: true, - buildBranch, - buildType: BUILD_TYPE.branchEventCause, - }; - }); + buildBranch: 'pr-123', + buildUrl, + }, + }; + }); - afterEach(() => { - jest.clearAllMocks(); - }); + afterEach(() => { + jest.clearAllMocks(); + }); - afterAll(() => { - jest.restoreAllMocks(); - }); + afterAll(() => { + jest.restoreAllMocks(); + }); - it('Uploads screenshots.', async () => { - const service = new WDIOTerraService({}, {}, config); - await service.onComplete(); - expect(upload).toHaveBeenCalled(); - expect(getRemoteScreenshotConfiguration).toHaveBeenCalledWith('screenshot sites object', buildBranch); - }); + it('Posts a comment', async () => { + getComments.mockResolvedValue(['not the warning message']); + postComment.mockResolvedValue(); + const service = new WDIOTerraService({}, {}, config); + await service.postMismatchWarningOnce(); + expect(postComment).toHaveBeenCalledWith(warningMessage); + }); - it('Does not upload if not using remote screenshots.', async () => { - config.serviceOptions.useRemoteReferenceScreenshots = false; - const service = new WDIOTerraService({}, {}, config); - await service.onComplete(); - expect(upload).not.toHaveBeenCalled(); - }); + it('Does not post if the comment is found on the issue', async () => { + getComments.mockResolvedValue([warningMessage]); + postComment.mockResolvedValue(); + const service = new WDIOTerraService({}, {}, config); + await service.postMismatchWarningOnce(); + expect(getComments).toHaveBeenCalled(); + expect(postComment).not.toHaveBeenCalled(); + }); - it('Does not upload if the build branch name matches the PR pattern.', async () => { - config.serviceOptions.buildBranch = 'pr-123'; - const service = new WDIOTerraService({}, {}, config); - await service.onComplete(); - expect(upload).not.toHaveBeenCalled(); - }); + it('Stops the service without posting if it fails to get the issue comments', async () => { + getComments.mockRejectedValue('oh no!'); + const service = new WDIOTerraService({}, {}, config); + await expect(service.postMismatchWarningOnce()).rejects.toEqual('oh no!'); + expect(getComments).toHaveBeenCalled(); + expect(postComment).not.toHaveBeenCalled(); + }); - it('Does not upload if the build type is not a branch event.', async () => { - config.serviceOptions.buildType = undefined; - const service = new WDIOTerraService({}, {}, config); - await service.onComplete(); - expect(upload).not.toHaveBeenCalled(); - }); + it('Stops the service if it fails to post the comment', async () => { + getComments.mockResolvedValue(['not the warning message']); + postComment.mockRejectedValue('oh no!'); + const service = new WDIOTerraService({}, {}, config); + await expect(service.postMismatchWarningOnce()).rejects.toEqual('oh no!'); + expect(getComments).toHaveBeenCalled(); + expect(postComment).toHaveBeenCalled(); + }); + }); - it('Stops the service if something goes wrong while uploading', async () => { - upload.mockRejectedValue('oh no!'); - const service = new WDIOTerraService({}, {}, config); - await expect(service.onComplete()).rejects.toThrow(SevereServiceError); - expect(upload).toHaveBeenCalled(); - }); + describe('uploadBuildBranchScreenshots function', () => { + let upload; + const buildBranch = 'not-a-pull-request'; + let config; + let getRemoteScreenshotConfiguration; + + beforeAll(() => { + getRemoteScreenshotConfiguration = jest.fn(() => ({ + publishScreenshotConfiguration: jest.fn(), + })); + upload = jest.spyOn(ScreenshotRequestor.prototype, 'upload'); + }); + + beforeEach(() => { + config = { + getRemoteScreenshotConfiguration, + screenshotsSites: 'screenshot sites object', + serviceOptions: { + useRemoteReferenceScreenshots: true, + buildBranch, + buildType: BUILD_TYPE.branchEventCause, + locale: 'en', + theme: 'terra-default-theme', + formFactor: 'large', + browser: 'chrome', + }, + }; + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + afterAll(() => { + jest.restoreAllMocks(); + }); + + it('Uploads screenshots.', async () => { + const service = new WDIOTerraService({}, {}, config); + await service.uploadBuildBranchScreenshots(); + expect(upload).toHaveBeenCalledWith('en', 'terra-default-theme', 'large', 'chrome'); + expect(getRemoteScreenshotConfiguration).toHaveBeenCalledWith('screenshot sites object', buildBranch); + }); + + it('Stops the service if something goes wrong while uploading', async () => { + upload.mockRejectedValue('oh no!'); + const service = new WDIOTerraService({}, {}, config); + await expect(service.uploadBuildBranchScreenshots()).rejects.toThrow(SevereServiceError); + expect(upload).toHaveBeenCalledWith('en', 'terra-default-theme', 'large', 'chrome'); }); }); }); From d1195e5fe8dd684149d9f317ce03a4443b31ce57 Mon Sep 17 00:00:00 2001 From: BenBoersma Date: Wed, 20 Dec 2023 11:56:28 -0600 Subject: [PATCH 05/10] adding changelog and fixing linter errors --- packages/terra-functional-testing/CHANGELOG.md | 5 +++++ .../jest/commands/expect/toMatchReference.test.js | 15 ++++++++------- .../commands/utils/ScreenshotRequestor.test.js | 4 ++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/packages/terra-functional-testing/CHANGELOG.md b/packages/terra-functional-testing/CHANGELOG.md index e52c7711f..0457dd3a2 100644 --- a/packages/terra-functional-testing/CHANGELOG.md +++ b/packages/terra-functional-testing/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +* Changed + * Updated upload and download logic in Nexus screenshots based on current locale, theme, browser, and formFactor + * Updated PR comment logic in Nexus screenshots. + * Updated Nexus mismatch warning to display testname and to be outputted via Logger + ## 4.4.0 - (September 26, 2023) * Changed diff --git a/packages/terra-functional-testing/tests/jest/commands/expect/toMatchReference.test.js b/packages/terra-functional-testing/tests/jest/commands/expect/toMatchReference.test.js index 9d1ade389..d0c6e049d 100644 --- a/packages/terra-functional-testing/tests/jest/commands/expect/toMatchReference.test.js +++ b/packages/terra-functional-testing/tests/jest/commands/expect/toMatchReference.test.js @@ -1,5 +1,6 @@ const path = require('path'); const fs = require('fs-extra'); + const mockInfo = jest.fn(); jest.mock('@cerner/terra-cli/lib/utils/Logger', () => function mock() { return { @@ -329,7 +330,7 @@ describe('toMatchReference', () => { const result = toMatchReference(receivedScreenshot, 'TestName'); const expectedMessage = 'Expected the screenshot to match the reference screenshot, but received a screenshot with different dimensions.\nExpected the screenshot to be within the mismatch tolerance, but received a mismatch difference of 0.1%.'; - + expect(result.pass).toBe(true); expect(result.message()).toEqual(expectedMessage); expect(mockInfo).toHaveBeenCalledWith('Test: \'TestName\' has a mismatch difference of 0.1% and needs to be reviewed.'); @@ -353,7 +354,7 @@ describe('toMatchReference', () => { const result = toMatchReference(receivedScreenshot, 'TestName'); const expectedMessage = 'Expected the screenshot to match the reference screenshot, but received a screenshot with different dimensions.\nExpected the screenshot to be within the mismatch tolerance, but received a mismatch difference of 0.1%.'; - + expect(result.pass).toBe(true); expect(fs.existsSync).toHaveBeenCalledTimes(1); expect(fs.mkdirSync).toHaveBeenCalledWith('/mock/', { recursive: true }); @@ -374,18 +375,18 @@ describe('toMatchReference', () => { isSameDimensions: false, misMatchPercentage: 0.10, }; - jest.spyOn(fs, 'existsSync').mockImplementation((pathName) => (pathName === '/mock/') ? true : false); + jest.spyOn(fs, 'existsSync').mockImplementation((pathName) => (pathName === '/mock/')); jest.spyOn(fs, 'writeFileSync').mockImplementationOnce(() => false); const result = toMatchReference(receivedScreenshot, 'TestName'); const expectedMessage = 'Expected the screenshot to match the reference screenshot, but received a screenshot with different dimensions.\nExpected the screenshot to be within the mismatch tolerance, but received a mismatch difference of 0.1%.'; - + expect(result.pass).toBe(true); expect(fs.existsSync).toHaveBeenCalledTimes(2); expect(fs.existsSync.mock.calls).toEqual([ - ['/mock/'], // First call - [path.join('/mock/', 'ignored-mismatch.json')] // Second call - ]) + ['/mock/'], + [path.join('/mock/', 'ignored-mismatch.json')], + ]); expect(fs.writeFileSync).toHaveBeenCalledWith(path.join('/mock/', 'ignored-mismatch.json'), JSON.stringify({ screenshotMismatched: true }, null, 2)); expect(result.message()).toEqual(expectedMessage); expect(mockInfo).toHaveBeenCalledWith('Test: \'TestName\' has a mismatch difference of 0.1% and needs to be reviewed.'); diff --git a/packages/terra-functional-testing/tests/jest/commands/utils/ScreenshotRequestor.test.js b/packages/terra-functional-testing/tests/jest/commands/utils/ScreenshotRequestor.test.js index 172c02db7..e587db2eb 100644 --- a/packages/terra-functional-testing/tests/jest/commands/utils/ScreenshotRequestor.test.js +++ b/packages/terra-functional-testing/tests/jest/commands/utils/ScreenshotRequestor.test.js @@ -397,8 +397,8 @@ describe('ScreenshotRequestor', () => { expect(ScreenshotRequestor.makeReferenceName).toHaveBeenCalledWith('locale', 'theme', 'formFactor', 'browser'); expect(screenshotRequestor.deleteExistingScreenshots).toHaveBeenCalledWith(referenceName); expect(screenshotRequestor.zipDirectoryToMemory).toHaveBeenCalled(); - expect(screenshotRequestor.uploadScreenshots).toHaveBeenCalledWith(memoryStream, referenceName) ; - expect(screenshotRequestor.deleteZippedLatestScreenshots).toHaveBeenCalled() ; + expect(screenshotRequestor.uploadScreenshots).toHaveBeenCalledWith(memoryStream, referenceName); + expect(screenshotRequestor.deleteZippedLatestScreenshots).toHaveBeenCalled(); ScreenshotRequestor.makeReferenceName = oldMakeReferenceName; ScreenshotRequestor.prototype.deleteExistingScreenshots = oldDeleteExistingScreenshots; ScreenshotRequestor.prototype.zipLatestScreenshots = oldZipLatestScreenshots; From 0f73f13811532ac2ef8026b400ae1551315c5ff9 Mon Sep 17 00:00:00 2001 From: BenBoersma Date: Tue, 2 Jan 2024 15:24:54 -0600 Subject: [PATCH 06/10] fixing falsy function call for making reference name --- .../src/commands/utils/ScreenshotRequestor.js | 9 +-------- .../jest/commands/utils/ScreenshotRequestor.test.js | 4 ++-- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/packages/terra-functional-testing/src/commands/utils/ScreenshotRequestor.js b/packages/terra-functional-testing/src/commands/utils/ScreenshotRequestor.js index 109ebbc3d..08c33f1a8 100644 --- a/packages/terra-functional-testing/src/commands/utils/ScreenshotRequestor.js +++ b/packages/terra-functional-testing/src/commands/utils/ScreenshotRequestor.js @@ -90,14 +90,7 @@ class ScreenshotRequestor { * @param {string} browser - the browser to use when uploading */ static makeReferenceName(locale, theme, formFactor, browser) { - const referenceName = [theme, locale, browser, formFactor].filter( - (str) => { - if (str !== '' || str !== undefined) { - return str; - } - return false; - }, - ).join('-'); + const referenceName = [theme, locale, browser, formFactor].filter((str) => str).join('-'); return referenceName; } diff --git a/packages/terra-functional-testing/tests/jest/commands/utils/ScreenshotRequestor.test.js b/packages/terra-functional-testing/tests/jest/commands/utils/ScreenshotRequestor.test.js index e587db2eb..879dc2e1a 100644 --- a/packages/terra-functional-testing/tests/jest/commands/utils/ScreenshotRequestor.test.js +++ b/packages/terra-functional-testing/tests/jest/commands/utils/ScreenshotRequestor.test.js @@ -413,8 +413,8 @@ describe('ScreenshotRequestor', () => { expect(ScreenshotRequestor.makeReferenceName('locale', 'theme', 'formFactor', 'browser')).toEqual('theme-locale-browser-formFactor'); }); - it('should should skip undefined arguments', () => { - expect(ScreenshotRequestor.makeReferenceName('locale', undefined, undefined, 'browser')).toEqual('locale-browser'); + it('should should skip falsy arguments', () => { + expect(ScreenshotRequestor.makeReferenceName('', 'locale', undefined, 'browser', null)).toEqual('locale-browser'); }); it('should return empty if no arguments are given', () => { From 3b352ec7520e8b85f08ff21aafbc8ae8aaa9a2fb Mon Sep 17 00:00:00 2001 From: BenBoersma Date: Tue, 16 Jan 2024 17:29:29 -0600 Subject: [PATCH 07/10] moving off sync functions in async oncomplete --- .../wdio-terra-service/WDIOTerraService.js | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/packages/terra-functional-testing/src/services/wdio-terra-service/WDIOTerraService.js b/packages/terra-functional-testing/src/services/wdio-terra-service/WDIOTerraService.js index 135a0409e..03e878bdb 100644 --- a/packages/terra-functional-testing/src/services/wdio-terra-service/WDIOTerraService.js +++ b/packages/terra-functional-testing/src/services/wdio-terra-service/WDIOTerraService.js @@ -266,19 +266,35 @@ class WDIOTerraService { const fileName = path.join(getOutputDir(), 'ignored-mismatch.json'); try { - if (fs.existsSync(fileName) && buildBranch.match(BUILD_BRANCH.pullRequest)) { - // We found a screenshot mismatch during our build of this PR branch. - await this.postMismatchWarningOnce(); - // Remove mismatch flag file after running - fs.removeSync(fileName); - } else if (!buildBranch.match(BUILD_BRANCH.pullRequest) && buildType === BUILD_TYPE.branchEventCause) { - // This non-PR branch is being merged or someone pushed code into it directly. - await this.uploadBuildBranchScreenshots(); - // Remove mismatch flag file after running if it exists - if (fs.existsSync(fileName)) { - fs.removeSync(fileName); + // Check if the file exists in the current directory. + fs.access(fileName, fs.constants.F_OK, async (error) => { + try { + if (!error && buildBranch.match(BUILD_BRANCH.pullRequest)) { + // We found a screenshot mismatch during our build of this PR branch. + await this.postMismatchWarningOnce(); + // Remove mismatch flag file after running + fs.unlink(fileName, (err) => { + if (err) throw err; + }); + } else if (!buildBranch.match(BUILD_BRANCH.pullRequest) && buildType === BUILD_TYPE.branchEventCause) { + // This non-PR branch is being merged or someone pushed code into it directly. + await this.uploadBuildBranchScreenshots(); + // Remove mismatch flag file after running if it exists + if (!error) { + fs.unlink(fileName, (err) => { + if (err) throw err; + }); + } + } + } catch (err) { + // The service will stop only if a SevereServiceError is thrown. + if (err instanceof SevereServiceError) { + throw err; + } + + throw new SevereServiceError(err); } - } + }); } catch (err) { // The service will stop only if a SevereServiceError is thrown. if (err instanceof SevereServiceError) { From 0fb4acdd8ae50dc3fdeb4084ca6b7241a0a686f3 Mon Sep 17 00:00:00 2001 From: BenBoersma Date: Wed, 17 Jan 2024 09:28:53 -0600 Subject: [PATCH 08/10] fixing lint error --- .../src/services/wdio-terra-service/WDIOTerraService.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/terra-functional-testing/src/services/wdio-terra-service/WDIOTerraService.js b/packages/terra-functional-testing/src/services/wdio-terra-service/WDIOTerraService.js index 03e878bdb..3725eb268 100644 --- a/packages/terra-functional-testing/src/services/wdio-terra-service/WDIOTerraService.js +++ b/packages/terra-functional-testing/src/services/wdio-terra-service/WDIOTerraService.js @@ -291,7 +291,7 @@ class WDIOTerraService { if (err instanceof SevereServiceError) { throw err; } - + throw new SevereServiceError(err); } }); From e3f1c358ea6ee77af9b877d072b9ca9f77df5871 Mon Sep 17 00:00:00 2001 From: BenBoersma Date: Wed, 17 Jan 2024 16:27:30 -0600 Subject: [PATCH 09/10] Used better testing methods and changed wdioterraservice to test new async calls --- .../commands/expect/toMatchReference.test.js | 15 +-- .../WDIOTerraService.test.js | 125 +++++++++++++----- 2 files changed, 96 insertions(+), 44 deletions(-) diff --git a/packages/terra-functional-testing/tests/jest/commands/expect/toMatchReference.test.js b/packages/terra-functional-testing/tests/jest/commands/expect/toMatchReference.test.js index d0c6e049d..6fc815131 100644 --- a/packages/terra-functional-testing/tests/jest/commands/expect/toMatchReference.test.js +++ b/packages/terra-functional-testing/tests/jest/commands/expect/toMatchReference.test.js @@ -260,8 +260,7 @@ describe('toMatchReference', () => { isSameDimensions: false, misMatchPercentage: 0.10, }; - jest.spyOn(fs, 'existsSync').mockImplementation(() => true); - jest.spyOn(fs, 'existsSync').mockImplementation(() => true); + jest.spyOn(fs, 'existsSync').mockReturnValue(true); const result = toMatchReference(receivedScreenshot, 'TestName'); const expectedMessage = 'Expected the screenshot to match the reference screenshot, but received a screenshot with different dimensions.\nExpected the screenshot to be within the mismatch tolerance, but received a mismatch difference of 0.1%.'; @@ -303,7 +302,7 @@ describe('toMatchReference', () => { isSameDimensions: false, misMatchPercentage: 0.10, }; - jest.spyOn(fs, 'existsSync').mockImplementation(() => true); + jest.spyOn(fs, 'existsSync').mockReturnValue(true); const result = toMatchReference(receivedScreenshot, 'TestName'); const expectedMessage = 'Expected the screenshot to match the reference screenshot, but received a screenshot with different dimensions.\nExpected the screenshot to be within the mismatch tolerance, but received a mismatch difference of 0.1%.'; @@ -326,7 +325,7 @@ describe('toMatchReference', () => { isSameDimensions: false, misMatchPercentage: 0.10, }; - jest.spyOn(fs, 'existsSync').mockImplementation(() => true); + jest.spyOn(fs, 'existsSync').mockReturnValue(true); const result = toMatchReference(receivedScreenshot, 'TestName'); const expectedMessage = 'Expected the screenshot to match the reference screenshot, but received a screenshot with different dimensions.\nExpected the screenshot to be within the mismatch tolerance, but received a mismatch difference of 0.1%.'; @@ -348,9 +347,9 @@ describe('toMatchReference', () => { isSameDimensions: false, misMatchPercentage: 0.10, }; - jest.spyOn(fs, 'existsSync').mockImplementationOnce(() => false); - jest.spyOn(fs, 'mkdirSync').mockImplementationOnce(() => false); - jest.spyOn(fs, 'writeFileSync').mockImplementationOnce(() => false); + jest.spyOn(fs, 'existsSync').mockReturnValueOnce(false); + jest.spyOn(fs, 'mkdirSync').mockReturnValueOnce(false); + jest.spyOn(fs, 'writeFileSync').mockReturnValueOnce(false); const result = toMatchReference(receivedScreenshot, 'TestName'); const expectedMessage = 'Expected the screenshot to match the reference screenshot, but received a screenshot with different dimensions.\nExpected the screenshot to be within the mismatch tolerance, but received a mismatch difference of 0.1%.'; @@ -376,7 +375,7 @@ describe('toMatchReference', () => { misMatchPercentage: 0.10, }; jest.spyOn(fs, 'existsSync').mockImplementation((pathName) => (pathName === '/mock/')); - jest.spyOn(fs, 'writeFileSync').mockImplementationOnce(() => false); + jest.spyOn(fs, 'writeFileSync').mockReturnValueOnce(false); const result = toMatchReference(receivedScreenshot, 'TestName'); const expectedMessage = 'Expected the screenshot to match the reference screenshot, but received a screenshot with different dimensions.\nExpected the screenshot to be within the mismatch tolerance, but received a mismatch difference of 0.1%.'; diff --git a/packages/terra-functional-testing/tests/jest/services/wdio-terra-service/WDIOTerraService.test.js b/packages/terra-functional-testing/tests/jest/services/wdio-terra-service/WDIOTerraService.test.js index 3546bf39c..6c7372695 100644 --- a/packages/terra-functional-testing/tests/jest/services/wdio-terra-service/WDIOTerraService.test.js +++ b/packages/terra-functional-testing/tests/jest/services/wdio-terra-service/WDIOTerraService.test.js @@ -239,7 +239,7 @@ describe('WDIO Terra Service', () => { }); }); - describe('onComplete hook', () => { + describe.only('onComplete hook', () => { let config; let getRemoteScreenshotConfiguration; let buildUrl; @@ -270,68 +270,100 @@ describe('WDIO Terra Service', () => { jest.restoreAllMocks(); }); - it('should call postMismatchWarningOnce if ignored-mismatch file exists and branch is pull-request', async () => { + it('should call postMismatchWarningOnce if ignored-mismatch file exists and branch is pull-request', async done => { const oldPostMismatchWarningOnce = WDIOTerraService.prototype.postMismatchWarningOnce; WDIOTerraService.prototype.postMismatchWarningOnce = jest.fn(); - jest.spyOn(fs, 'existsSync').mockImplementation(() => true); - jest.spyOn(fs, 'removeSync').mockImplementation(); + jest.spyOn(fs, 'access').mockImplementation(async (path, opts, cb) => { + expect(path).toBe('/mock/ignored-mismatch.json'); + expect(opts).toBe(fs.constants.F_OK); + expect(cb).toBeDefined(); + await cb(); + }); + jest.spyOn(fs, 'unlink').mockImplementation(async (path, cb) => { + expect(path).toBe('/mock/ignored-mismatch.json'); + expect(cb).toBeDefined(); + await cb(); + done(); + }); config.serviceOptions.buildBranch = 'pr-123'; const service = new WDIOTerraService({}, {}, config); service.postMismatchWarningOnce.mockResolvedValueOnce(); await service.onComplete(); - expect(fs.existsSync).toHaveBeenCalledWith('/mock/ignored-mismatch.json'); expect(service.postMismatchWarningOnce).toHaveBeenCalled(); - expect(fs.removeSync).toHaveBeenCalledWith('/mock/ignored-mismatch.json'); WDIOTerraService.prototype.postMismatchWarningOnce = oldPostMismatchWarningOnce; config.serviceOptions.buildBranch = undefined; }); - it('should not call postMismatchWarningOnce if ignored-mismatch file does not exist', async () => { + it('should not call postMismatchWarningOnce if ignored-mismatch file does not exist', async done => { const oldPostMismatchWarningOnce = WDIOTerraService.prototype.postMismatchWarningOnce; WDIOTerraService.prototype.postMismatchWarningOnce = jest.fn(); - jest.spyOn(fs, 'existsSync').mockImplementation(() => false); - jest.spyOn(fs, 'removeSync').mockImplementation(); + jest.spyOn(fs, 'access').mockImplementation(async (path, opts, cb) => { + expect(path).toBe('/mock/ignored-mismatch.json'); + expect(opts).toBe(fs.constants.F_OK); + expect(cb).toBeDefined(); + await cb(new Error('File does not exist')); + done(); + }); + jest.spyOn(fs, 'unlink').mockImplementation(async (path, cb) => { + await cb(); + done(new Error()); + }); config.serviceOptions.buildBranch = 'pr-123'; const service = new WDIOTerraService({}, {}, config); service.postMismatchWarningOnce.mockResolvedValueOnce(); await service.onComplete(); - expect(fs.existsSync).toHaveBeenCalledWith('/mock/ignored-mismatch.json'); expect(service.postMismatchWarningOnce).not.toHaveBeenCalled(); - expect(fs.removeSync).not.toHaveBeenCalled(); + expect(fs.unlink).not.toHaveBeenCalled(); WDIOTerraService.prototype.postMismatchWarningOnce = oldPostMismatchWarningOnce; config.serviceOptions.buildBranch = undefined; }); - it('should not call postMismatchWarningOnce if branch is not pull-request', async () => { + it('should not call postMismatchWarningOnce if branch is not pull-request', async done => { const oldPostMismatchWarningOnce = WDIOTerraService.prototype.postMismatchWarningOnce; WDIOTerraService.prototype.postMismatchWarningOnce = jest.fn(); - jest.spyOn(fs, 'existsSync').mockImplementation(() => true); - jest.spyOn(fs, 'removeSync').mockImplementation(); + jest.spyOn(fs, 'access').mockImplementation(async (path, opts, cb) => { + expect(path).toBe('/mock/ignored-mismatch.json'); + expect(opts).toBe(fs.constants.F_OK); + expect(cb).toBeDefined(); + await cb(); + done(); + }); + jest.spyOn(fs, 'unlink').mockImplementation(async (path, cb) => { + await cb(); + done(new Error()); + }); config.serviceOptions.buildBranch = 'master'; const service = new WDIOTerraService({}, {}, config); service.postMismatchWarningOnce.mockResolvedValueOnce(); await service.onComplete(); - expect(fs.existsSync).toHaveBeenCalledWith('/mock/ignored-mismatch.json'); expect(service.postMismatchWarningOnce).not.toHaveBeenCalled(); - expect(fs.removeSync).not.toHaveBeenCalled(); WDIOTerraService.prototype.postMismatchWarningOnce = oldPostMismatchWarningOnce; config.serviceOptions.buildBranch = undefined; }); - it('should call uploadBuildBranchScreenshots if build type is a commit and it is not a pull request', async () => { + it('should call uploadBuildBranchScreenshots if build type is a commit and it is not a pull request', async done => { const oldUploadBuildBranchScreenshots = WDIOTerraService.prototype.uploadBuildBranchScreenshots; WDIOTerraService.prototype.uploadBuildBranchScreenshots = jest.fn(); - jest.spyOn(fs, 'existsSync').mockImplementation(() => false); - jest.spyOn(fs, 'removeSync').mockImplementation(); + jest.spyOn(fs, 'access').mockImplementation(async (path, opts, cb) => { + expect(path).toBe('/mock/ignored-mismatch.json'); + expect(opts).toBe(fs.constants.F_OK); + expect(cb).toBeDefined(); + await cb(new Error('File does not exist')); + done(); + }); + jest.spyOn(fs, 'unlink').mockImplementation(async (path, cb) => { + await cb(); + done(new Error()); + }); config.serviceOptions.buildBranch = 'master'; config.serviceOptions.buildType = BUILD_TYPE.branchEventCause; const service = new WDIOTerraService({}, {}, config); @@ -339,19 +371,28 @@ describe('WDIO Terra Service', () => { await service.onComplete(); - expect(fs.existsSync).toHaveBeenCalledWith('/mock/ignored-mismatch.json'); expect(service.uploadBuildBranchScreenshots).toHaveBeenCalled(); - expect(fs.removeSync).not.toHaveBeenCalled(); WDIOTerraService.prototype.uploadBuildBranchScreenshots = oldUploadBuildBranchScreenshots; config.serviceOptions.buildBranch = undefined; }); - it('should call uploadBuildBranchScreenshots and remove the ignored-mismatch file if it exists', async () => { + it('should call uploadBuildBranchScreenshots and remove the ignored-mismatch file if it exists', async done => { const oldUploadBuildBranchScreenshots = WDIOTerraService.prototype.uploadBuildBranchScreenshots; WDIOTerraService.prototype.uploadBuildBranchScreenshots = jest.fn(); - jest.spyOn(fs, 'existsSync').mockImplementation(() => true); - jest.spyOn(fs, 'removeSync').mockImplementation(); + jest.spyOn(fs, 'access').mockImplementation(async (path, opts, cb) => { + expect(path).toBe('/mock/ignored-mismatch.json'); + expect(opts).toBe(fs.constants.F_OK); + expect(cb).toBeDefined(); + await cb(); + done(); + }); + jest.spyOn(fs, 'unlink').mockImplementation(async (path, cb) => { + expect(path).toBe('/mock/ignored-mismatch.json'); + expect(cb).toBeDefined(); + await cb(); + done(); + }); config.serviceOptions.buildBranch = 'master'; config.serviceOptions.buildType = BUILD_TYPE.branchEventCause; const service = new WDIOTerraService({}, {}, config); @@ -359,19 +400,26 @@ describe('WDIO Terra Service', () => { await service.onComplete(); - expect(fs.existsSync).toHaveBeenCalledWith('/mock/ignored-mismatch.json'); expect(service.uploadBuildBranchScreenshots).toHaveBeenCalled(); - expect(fs.removeSync).toHaveBeenCalledWith('/mock/ignored-mismatch.json'); WDIOTerraService.prototype.uploadBuildBranchScreenshots = oldUploadBuildBranchScreenshots; config.serviceOptions.buildBranch = undefined; }); - it('should not call uploadBuildBranchScreenshots if build is a PR', async () => { + it('should not call uploadBuildBranchScreenshots if build is a PR', async done => { const oldUploadBuildBranchScreenshots = WDIOTerraService.prototype.uploadBuildBranchScreenshots; WDIOTerraService.prototype.uploadBuildBranchScreenshots = jest.fn(); - jest.spyOn(fs, 'existsSync').mockImplementation(() => false); - jest.spyOn(fs, 'removeSync').mockImplementation(); + jest.spyOn(fs, 'access').mockImplementation(async (path, opts, cb) => { + expect(path).toBe('/mock/ignored-mismatch.json'); + expect(opts).toBe(fs.constants.F_OK); + expect(cb).toBeDefined(); + await cb(new Error('File does not exist')); + done(); + }); + jest.spyOn(fs, 'unlink').mockImplementation(async (path, cb) => { + await cb(); + done(new Error()); + }); config.serviceOptions.buildBranch = 'pr-123'; config.serviceOptions.buildType = BUILD_TYPE.branchEventCause; const service = new WDIOTerraService({}, {}, config); @@ -379,19 +427,26 @@ describe('WDIO Terra Service', () => { await service.onComplete(); - expect(fs.existsSync).toHaveBeenCalledWith('/mock/ignored-mismatch.json'); expect(service.uploadBuildBranchScreenshots).not.toHaveBeenCalled(); - expect(fs.removeSync).not.toHaveBeenCalled(); WDIOTerraService.prototype.uploadBuildBranchScreenshots = oldUploadBuildBranchScreenshots; config.serviceOptions.buildBranch = undefined; }); - it('should not call uploadBuildBranchScreenshots if build type is not a branchEventCause', async () => { + it('should not call uploadBuildBranchScreenshots if build type is not a branchEventCause', async done => { const oldUploadBuildBranchScreenshots = WDIOTerraService.prototype.uploadBuildBranchScreenshots; WDIOTerraService.prototype.uploadBuildBranchScreenshots = jest.fn(); - jest.spyOn(fs, 'existsSync').mockImplementation(() => false); - jest.spyOn(fs, 'removeSync').mockImplementation(); + jest.spyOn(fs, 'access').mockImplementation(async (path, opts, cb) => { + expect(path).toBe('/mock/ignored-mismatch.json'); + expect(opts).toBe(fs.constants.F_OK); + expect(cb).toBeDefined(); + await cb(new Error('File does not exist')); + done(); + }); + jest.spyOn(fs, 'unlink').mockImplementation(async (path, cb) => { + await cb(); + done(new Error()); + }); config.serviceOptions.buildBranch = 'master'; config.serviceOptions.buildType = 'Replayed'; const service = new WDIOTerraService({}, {}, config); @@ -399,9 +454,7 @@ describe('WDIO Terra Service', () => { await service.onComplete(); - expect(fs.existsSync).toHaveBeenCalledWith('/mock/ignored-mismatch.json'); expect(service.uploadBuildBranchScreenshots).not.toHaveBeenCalled(); - expect(fs.removeSync).not.toHaveBeenCalled(); WDIOTerraService.prototype.uploadBuildBranchScreenshots = oldUploadBuildBranchScreenshots; config.serviceOptions.buildBranch = undefined; From 8bbf50aa8f528ec250895077dee925837e7c24be Mon Sep 17 00:00:00 2001 From: Saad Adnan <38024451+sdadn@users.noreply.github.com> Date: Wed, 17 Jan 2024 17:26:39 -0600 Subject: [PATCH 10/10] Update CHANGELOG.md --- packages/terra-functional-testing/CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/terra-functional-testing/CHANGELOG.md b/packages/terra-functional-testing/CHANGELOG.md index 681bdbbf0..1f97f0393 100644 --- a/packages/terra-functional-testing/CHANGELOG.md +++ b/packages/terra-functional-testing/CHANGELOG.md @@ -3,9 +3,9 @@ ## Unreleased * Changed - * Updated upload and download logic in Nexus screenshots based on current locale, theme, browser, and formFactor + * Updated upload and download logic in Nexus screenshots based on current locale, theme, browser, and formFactor. * Updated PR comment logic in Nexus screenshots. - * Updated Nexus mismatch warning to display testname and to be outputted via Logger + * Updated Nexus mismatch warning to display testname and to be outputted via Logger. ## 4.5.0 - (December 11, 2023)