diff --git a/packages/app/src/runner/event-manager.ts b/packages/app/src/runner/event-manager.ts index eb02090ad72f..4d8cc4b5062b 100644 --- a/packages/app/src/runner/event-manager.ts +++ b/packages/app/src/runner/event-manager.ts @@ -390,6 +390,15 @@ export class EventManager { window.Cypress = Cypress this._addListeners() + + // let actions: any + + // await new Promise((resolve, reject) => { + // this.ws.emit('burn:in:actions', (res) => { + // actions = res + // resolve(null) + // }) + // }) } isBrowser (browserName) { @@ -398,9 +407,17 @@ export class EventManager { return this.Cypress.isBrowser(browserName) } - initialize ($autIframe: JQuery, config: Record) { + async initialize ($autIframe: JQuery, config: Record) { performance.mark('initialize-start') + const actions = await Cypress.backend('burn:in:actions') + + window.burnInActions = actions + + console.log('==========================================================================================================================================================================================================================================') + console.log(JSON.stringify(actions, null, 2)) + // console.log('==========================================================================================================================================================================================================================================') + const testFilter = this.specStore.testFilter return Cypress.initialize({ diff --git a/packages/app/src/runner/index.ts b/packages/app/src/runner/index.ts index 69972208c2ca..0eb02a48df88 100644 --- a/packages/app/src/runner/index.ts +++ b/packages/app/src/runner/index.ts @@ -231,7 +231,7 @@ export function addCrossOriginIframe (location) { * Cypress on it. * */ -function runSpecCT (config, spec: SpecFile) { +async function runSpecCT (config, spec: SpecFile) { const $runnerRoot = getRunnerElement() // clear AUT, if there is one. @@ -261,7 +261,7 @@ function runSpecCT (config, spec: SpecFile) { $autIframe.prop('src', specSrc) // initialize Cypress (driver) with the AUT! - getEventManager().initialize($autIframe, config) + await getEventManager().initialize($autIframe, config) } /** @@ -292,7 +292,7 @@ function setSpecForDriver (spec: SpecFile) { * a Spec IFrame to load the spec's source code, and * initialize Cypress on the AUT. */ -function runSpecE2E (config, spec: SpecFile) { +async function runSpecE2E (config, spec: SpecFile) { const $runnerRoot = getRunnerElement() // clear AUT, if there is one. @@ -340,7 +340,7 @@ function runSpecE2E (config, spec: SpecFile) { }) // initialize Cypress (driver) with the AUT! - getEventManager().initialize($autIframe, config) + await getEventManager().initialize($autIframe, config) } export function getRunnerConfigFromWindow () { @@ -433,11 +433,11 @@ async function executeSpec (spec: SpecFile, isRerun: boolean = false) { await getEventManager().setup(config) if (window.__CYPRESS_TESTING_TYPE__ === 'e2e') { - return runSpecE2E(config, spec) + return await runSpecE2E(config, spec) } if (window.__CYPRESS_TESTING_TYPE__ === 'component') { - return runSpecCT(config, spec) + return await runSpecCT(config, spec) } throw Error('Unknown or undefined testingType on window.__CYPRESS_TESTING_TYPE__') diff --git a/packages/driver/src/cypress/mocha.ts b/packages/driver/src/cypress/mocha.ts index e586dc514391..318ea7994b5d 100644 --- a/packages/driver/src/cypress/mocha.ts +++ b/packages/driver/src/cypress/mocha.ts @@ -1,3 +1,4 @@ +console.log('root 2') /* eslint-disable prefer-rest-params */ import _ from 'lodash' import $errUtils, { CypressError } from './error_utils' @@ -37,10 +38,29 @@ delete (window as any).Mocha export const SKIPPED_DUE_TO_BROWSER_MESSAGE = ' (skipped due to browser)' +type LatestScore = null | -2 | -1 | 0 | 1 + +type AttemptStrategy = 'RETRY' | 'BURN_IN' | 'NONE' + +type ReasonToStop = +| 'PASSED_FIRST_ATTEMPT' // no burn-in needed +| 'PASSED_BURN_IN' // achieved burn-in +| 'PASSED_MET_THRESHOLD' // passed after reaching threshold for strategy 'detect-flake-and-pass-on-threshold' +| 'FAILED_NO_RETRIES' // failed and no retries +| 'FAILED_REACHED_MAX_RETRIES' // failed after reaching max retries +| 'FAILED_DID_NOT_MEET_THRESHOLD' // failed since it's impossible to meet threshold for strategy 'detect-flake-and-pass-on-threshold' +| 'FAILED_STOPPED_ON_FLAKE' // failed with one attempt passing and using strategy 'detect-flake-but-always-fail' with `stopIfAnyPassed` set to true +// NOTE: can we detect this? how? the goal is to avoid retrying a test that failed because of a hook failure +| 'FAILED_HOOK_FAILED' // failed because a hook failed + interface CypressTest extends Mocha.Test { prevAttempts: CypressTest[] final?: boolean forceState?: 'passed' + latestScore?: LatestScore + thisAttemptInitialStrategy?: AttemptStrategy + nextAttemptStrategy?: AttemptStrategy + reasonToStop?: ReasonToStop } type Strategy = 'detect-flake-and-pass-on-threshold' | 'detect-flake-but-always-fail' | undefined @@ -56,8 +76,31 @@ type Options = T extends 'detect-flake-and-pass-on-threshold' ? } : undefined +type CompleteBurnInConfig = { + enabled: boolean + default: number + flaky: number +} + +function getNeededBurnInAttempts (latestScore: LatestScore, burnInConfig: CompleteBurnInConfig) { + if (burnInConfig.enabled === false) { + return 0 + } + + switch (latestScore) { + case null: return burnInConfig.default // this means the cloud determined the test is new or modified + case 0: return burnInConfig.default // this means the cloud determined the test was failing with no flake + case -1: return burnInConfig.flaky // this means the cloud determined the test was flaky + case -2: return 0 // this means the cloud couldn't determine the score + case 1: return 0 // this means the cloud determined the test graduated burn-in + default: return 0 + } +} + // NOTE: 'calculateTestStatus' is marked as an individual function to make functionality easier to test. -export function calculateTestStatus (test: CypressTest, strategy: Strategy, options: Options) { +export function calculateTestStatus (test: CypressTest, strategy: Strategy, options: Options, completeBurnInConfig: CompleteBurnInConfig, latestScore: 0 | 1 | -1 | -2 | null) { + const neededBurnInAttempts = 0 //getNeededBurnInAttempts(latestScore, completeBurnInConfig) + // @ts-expect-error const totalAttemptsAlreadyExecuted = test.currentRetry() + 1 let shouldAttemptsContinue: boolean = true @@ -73,10 +116,17 @@ export function calculateTestStatus (test: CypressTest, strategy: Strategy, opti failedTests.push(test) } + if (!test.prevAttempts?.length) { + test.thisAttemptInitialStrategy = 'NONE' + } else { + test.thisAttemptInitialStrategy = test.prevAttempts[test.prevAttempts.length - 1].nextAttemptStrategy + } + // If there is AT LEAST one failed test attempt, we know we need to apply retry logic. // Otherwise, the test might be burning in (not implemented yet) OR the test passed on the first attempt, // meaning retry logic does NOT need to be applied. if (failedTests.length > 0) { + test.nextAttemptStrategy = 'RETRY' const maxAttempts = test.retries() + 1 const remainingAttempts = maxAttempts - totalAttemptsAlreadyExecuted const passingAttempts = passedTests.length @@ -95,40 +145,68 @@ export function calculateTestStatus (test: CypressTest, strategy: Strategy, opti ((options as Options<'detect-flake-but-always-fail'>).stopIfAnyPassed || false) : null - // Do we have the required amount of passes? If yes, we no longer need to keep running the test. - if (strategy !== 'detect-flake-but-always-fail' && passingAttempts >= (passesRequired as number)) { - outerTestStatus = 'passed' - test.final = true - shouldAttemptsContinue = false - } else if (totalAttemptsAlreadyExecuted < maxAttempts && - ( - // For strategy "detect-flake-and-pass-on-threshold" or no strategy (current GA retries): - // If we haven't met our max attempt limit AND we have enough remaining attempts that can satisfy the passing requirement. - // retry the test. - (strategy !== 'detect-flake-but-always-fail' && remainingAttempts >= (neededPassingAttemptsLeft as number)) || - // For strategy "detect-flake-but-always-fail": - // If we haven't met our max attempt limit AND - // stopIfAnyPassed is false OR - // stopIfAnyPassed is true and no tests have passed yet. - // retry the test. - (strategy === 'detect-flake-but-always-fail' && (!stopIfAnyPassed || stopIfAnyPassed && passingAttempts === 0)) - )) { - test.final = false + switch (strategy) { + case 'detect-flake-and-pass-on-threshold': + if (passingAttempts >= (passesRequired as number)) { + // we met the threshold, so we can stop retrying and pass the test + outerTestStatus = 'passed' + test.final = true + shouldAttemptsContinue = false + test.reasonToStop = 'PASSED_MET_THRESHOLD' + } else if (remainingAttempts < (neededPassingAttemptsLeft as number)) { + // we don't have enough remaining attempts to meet the threshold, so we should stop retrying and fail the test + outerTestStatus = 'failed' + test.final = true + test.forceState = test.state === 'passed' ? test.state : undefined + shouldAttemptsContinue = false + test.reasonToStop = 'FAILED_DID_NOT_MEET_THRESHOLD' + } else { + // we haven't met the threshold, but we have enough remaining attempts to meet the threshold, so we should retry the test + test.final = false + shouldAttemptsContinue = true + } + + break + case 'detect-flake-but-always-fail': + if (stopIfAnyPassed && passingAttempts > 0) { + // we have a passing attempt and we should stop retrying and fail the test + outerTestStatus = 'failed' + test.final = true + test.forceState = test.state === 'passed' ? test.state : undefined + shouldAttemptsContinue = false + test.reasonToStop = 'FAILED_STOPPED_ON_FLAKE' + } else if (remainingAttempts === 0) { + // we have no remaining attempts and we should stop retrying and fail the test + outerTestStatus = 'failed' + test.final = true + test.forceState = test.state === 'passed' ? test.state : undefined + shouldAttemptsContinue = false + test.reasonToStop = 'FAILED_REACHED_MAX_RETRIES' + } else { + // we have remaining attempts and we should retry the test + test.final = false + shouldAttemptsContinue = true + } + + break + default: + outerTestStatus = 'failed' + test.final = true + test.forceState = test.state === 'passed' ? test.state : undefined + shouldAttemptsContinue = false + test.reasonToStop = 'FAILED_NO_RETRIES' + } + } else { + test.nextAttemptStrategy = 'BURN_IN' + if (neededBurnInAttempts > passedTests.length) { shouldAttemptsContinue = true + test.final = false } else { - // Otherwise, we should stop retrying the test. - outerTestStatus = 'failed' - test.final = true - // If an outerStatus is 'failed', but the last test attempt was 'passed', we need to force the status so mocha doesn't flag the test attempt as failed. - // This is a common use case with 'detect-flake-but-always-fail', where we want to display the last attempt as 'passed' but fail the test. - test.forceState = test.state === 'passed' ? test.state : undefined + test.reasonToStop = neededBurnInAttempts > 0 ? 'PASSED_BURN_IN' : 'PASSED_FIRST_ATTEMPT' + outerTestStatus = 'passed' shouldAttemptsContinue = false + test.final = true } - } else { - // retry logic did not need to be applied and the test passed. - outerTestStatus = 'passed' - shouldAttemptsContinue = false - test.final = true } return { @@ -438,16 +516,38 @@ function patchTestClone () { } } -function createCalculateTestStatus (Cypress: Cypress.Cypress) { +function createCalculateTestStatus (cy: Cypress.Cypress) { // Adds a method to the test object called 'calculateTestStatus' // which is used inside our mocha patch (./driver/patches/mocha+7.0.1.dev.patch) // in order to calculate test retries. This prototype functions as a light abstraction around // 'calculateTestStatus', which makes the function easier to unit-test Test.prototype.calculateTestStatus = function () { - let retriesConfig = Cypress.config('retries') + let retriesConfig = cy.config('retries') + //const testConfig = originalConfig.randomKey + + const burnInActions = window.burnInActions + + console.log({ burnInActions }) + // @ts-expect-error + // const burnInConfig = cy.config('experimentalBurnIn') + + // let burnInAction: any = null + + console.log('point 1') + // debugger + + // @ts-expect-error + const burnInAction = cy.backend('burnIn:action', this.id, burnInConfig) + + console.log('point 2') + // debugger + + // const startingScore = burnInAction.startingScore + + // const completeBurnInConfig = burnInAction.completeBurnInConfig // @ts-expect-error - return calculateTestStatus(this, retriesConfig?.experimentalStrategy, retriesConfig?.experimentalOptions) + return calculateTestStatus(this, retriesConfig?.experimentalStrategy, retriesConfig?.experimentalOptions, { default: 3, flaky: 5, enabled: true }, -1) } } diff --git a/packages/server/lib/modes/record.js b/packages/server/lib/modes/record.js index 01f90aa46105..1402234a6a85 100644 --- a/packages/server/lib/modes/record.js +++ b/packages/server/lib/modes/record.js @@ -1084,6 +1084,13 @@ const createRunAndRecordSpecs = (options = {}) => { return } + const burnInActions = _.chain(response.actions).filter({ action: 'BURN_IN' }).map((x) => _.omit(x, ['action', 'type'])).value() + + if (_.some(response.actions, { action: 'BURN_IN' })) { + debug(`received burn in actions %o`, burnInActions) + project.setBurnInActions(burnInActions) + } + return cb(response) }) diff --git a/packages/server/lib/project-base.ts b/packages/server/lib/project-base.ts index c8d879b27f3b..2d2b4e793710 100644 --- a/packages/server/lib/project-base.ts +++ b/packages/server/lib/project-base.ts @@ -18,7 +18,7 @@ import { SocketE2E } from './socket-e2e' import { ensureProp } from './util/class-helpers' import system from './util/system' -import type { BannersState, FoundBrowser, FoundSpec, OpenProjectLaunchOptions, ReceivedCypressOptions, ResolvedConfigurationOptions, TestingType, VideoRecording } from '@packages/types' +import type { BannersState, FoundBrowser, FoundSpec, OpenProjectLaunchOptions, ReceivedCypressOptions, ResolvedConfigurationOptions, TestingType, VideoRecording, BurnInAction } from '@packages/types' import { DataContext, getCtx } from '@packages/data-context' import { createHmac } from 'crypto' import type ProtocolManager from './cloud/protocol' @@ -44,6 +44,7 @@ export interface Cfg extends ReceivedCypressOptions { component: Partial additionalIgnorePattern?: string | string[] resolved: ResolvedConfigurationOptions + randomKey: string[] } const localCwd = process.cwd() @@ -70,6 +71,7 @@ export class ProjectBase extends EE { public testingType: Cypress.TestingType public spec: FoundSpec | null public isOpen: boolean = false + private _burnInActions: BurnInAction[] = [] projectRoot: string constructor ({ @@ -430,6 +432,10 @@ export class ProjectBase extends EE { this.browser = browser } + setBurnInActions (actions: BurnInAction[]) { + this._server?.setBurnInActions(actions) + } + get protocolManager (): ProtocolManager | undefined { return this._protocolManager } diff --git a/packages/server/lib/server-base.ts b/packages/server/lib/server-base.ts index 78cfbd7a9d6d..24bfd5636b8f 100644 --- a/packages/server/lib/server-base.ts +++ b/packages/server/lib/server-base.ts @@ -28,7 +28,7 @@ import { createInitialWorkers } from '@packages/rewriter' import type { Cfg } from './project-base' import type { Browser } from '@packages/server/lib/browsers/types' import { InitializeRoutes, createCommonRoutes } from './routes' -import type { FoundSpec, ProtocolManagerShape, TestingType } from '@packages/types' +import type { BurnInAction, FoundSpec, ProtocolManagerShape, TestingType } from '@packages/types' import type { Server as WebSocketServer } from 'ws' import { RemoteStates } from './remote_states' import { cookieJar, SerializableAutomationCookie } from './util/cookies' @@ -218,6 +218,10 @@ export class ServerBase { this._networkProxy?.setProtocolManager(protocolManager) } + setBurnInActions (actions: BurnInAction[]) { + this._socket?.setBurnInActions(actions) + } + setupCrossOriginRequestHandling () { this._eventBus.on('cross:origin:cookies', (cookies: SerializableAutomationCookie[]) => { this.socket.localBus.once('cross:origin:cookies:received', () => { diff --git a/packages/server/lib/socket-base.ts b/packages/server/lib/socket-base.ts index bff626cf5dff..63a1bde0e97a 100644 --- a/packages/server/lib/socket-base.ts +++ b/packages/server/lib/socket-base.ts @@ -25,7 +25,8 @@ import { telemetry } from '@packages/telemetry' // eslint-disable-next-line no-duplicate-imports import type { Socket } from '@packages/socket' -import type { RunState, CachedTestState, ProtocolManagerShape } from '@packages/types' +import type { RunState, CachedTestState, ProtocolManagerShape, BurnInAction } from '@packages/types' +import { BurnInManager, getBurnInConfig, mergeBurnInConfig } from '@packages/types' import { cors } from '@packages/network' import memory from './browsers/memory' import { privilegedCommandsManager } from './privileged-commands/privileged-commands-manager' @@ -35,6 +36,7 @@ type StartListeningCallbacks = { } const debug = Debug('cypress:server:socket-base') +const debugBurnIn = Debug('cypress:server:burn-in') const retry = (fn: (res: any) => void) => { return Bluebird.delay(25).then(fn) @@ -52,6 +54,7 @@ export class SocketBase { protected ended: boolean protected _socketIo?: socketIo.SocketIOServer protected _cdpIo?: CDPSocketServer + protected _burnInManager: BurnInManager localBus: EventEmitter constructor (config: Record) { @@ -59,6 +62,7 @@ export class SocketBase { this.supportsRunEvents = config.isTextTerminal || config.experimentalInteractiveRunEvents this.ended = false this.localBus = new EventEmitter() + this._burnInManager = new BurnInManager() } protected ensureProp = ensureProp @@ -478,6 +482,27 @@ export class SocketBase { return (telemetry.exporter() as OTLPTraceExporterCloud)?.send(args[0], () => {}, (err) => { debug('error exporting telemetry data from browser %s', err) }) + case 'burn:in:actions': + { + debugBurnIn('burn:in:actions') + + return this._burnInManager.getActions() + // const id = args[0] + // const action = this._burnInManager.getActionDetails(id) + + // const burnInConfig = args[1] + // const localBurnInConfig = getBurnInConfig(burnInConfig) + + // const completeBurnInConfig = mergeBurnInConfig(action?.config ?? {}, { values: localBurnInConfig }) + + // debugBurnIn('completeBurnInConfig %o', completeBurnInConfig) + + // const result = { completeBurnInConfig, startingScore: action ? action.startingScore : -2 } + + // debugBurnIn('Getting action for %o, result: %o', id, result) + + // return result + } default: throw new Error(`You requested a backend event we cannot handle: ${eventName}`) } @@ -656,4 +681,12 @@ export class SocketBase { setProtocolManager (protocolManager: ProtocolManagerShape | undefined) { this._protocolManager = protocolManager } + + setBurnInActions (actions: BurnInAction[]) { + this._burnInManager?.setActions(actions) + } + + getBurnInActions () { + return this._burnInManager?.getActions() + } } diff --git a/packages/socket/lib/types.ts b/packages/socket/lib/types.ts index d5212cd6b453..4bc0416a9b84 100644 --- a/packages/socket/lib/types.ts +++ b/packages/socket/lib/types.ts @@ -1,3 +1,4 @@ +import type { BurnInAction } from '@packages/types' import type Emitter from 'component-emitter' -export type SocketShape = Emitter +export type SocketShape = Emitter & { getBurnInActions(): BurnInAction[] } diff --git a/packages/types/src/burnIn.ts b/packages/types/src/burnIn.ts new file mode 100644 index 000000000000..a14291ee3c28 --- /dev/null +++ b/packages/types/src/burnIn.ts @@ -0,0 +1,75 @@ +import { merge, clone } from 'lodash' + +export type BurnInConfig = { + default?: number | undefined + enabled?: boolean | undefined + flaky?: number | undefined +} + +export type BurnInConfigInstructions = { + overrides?: BurnInConfig + values?: BurnInConfig +} + +export type BurnInActionPayload = { + config: BurnInConfigInstructions + startingScore: 0 | 1 | -1 | -2 | null + planType: string +} + +export type BurnInAction = { + clientId: string | null + payload: BurnInActionPayload +} + +export type UserFacingBurnInConfig = boolean | { + default: number + flaky: number +} + +export class BurnInManager { + private _actions: BurnInAction[] = [] + + setActions (actions: BurnInAction[]) { + this._actions = actions + } + + getActionDetails (clientId: string | null) { + if (!clientId) return null + + return this._actions.find((action) => action.clientId === clientId)?.payload ?? null + } + + getActions () { + return clone(this._actions) + } +} + +export function getBurnInConfig (burnInConfig: UserFacingBurnInConfig): BurnInConfig { + return typeof burnInConfig === 'boolean' ? { enabled: burnInConfig } : { enabled: true, ...burnInConfig } +} + +export function mergeBurnInConfig ( + layer1Config: + | BurnInConfigInstructions + | BurnInConfig, + layer2Config: + | BurnInConfigInstructions + | BurnInConfig, +) { + const layer1Overrides = + 'overrides' in layer1Config ? layer1Config.overrides ?? {} : {} + const layer1Values = + 'values' in layer1Config ? layer1Config.values ?? {} : (layer1Config as BurnInConfig) + + const layer2Overrides = + 'overrides' in layer2Config ? layer2Config.overrides ?? {} : {} + const layer2Values = + 'values' in layer2Config ? layer2Config.values ?? {} : (layer2Config as BurnInConfig) + + const overrides = merge(layer2Overrides, layer1Overrides) // precedence is given to layer1 overrides + const combinedValues = merge(layer1Values, layer2Values) // precedence is given to layer2 values + const result = merge(combinedValues, overrides) // precedence is given to overrides + + return result +} diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 5ba54eda0e31..b5d859cdf1d0 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -16,6 +16,8 @@ export * from './auth' export * from './browser' +export * from './burnIn' + export type { PlatformName } from './platform' export {