feat: add burn-in to test execution logic

packages/app/src/runner/event-manager.ts

@@ -390,6 +390,15 @@ export class EventManager {
     window.Cypress = Cypress
 
     this._addListeners()
+
+    // let actions: any
+
+    // await new Promise((resolve, reject) => {
+    //   this.ws.emit('burn:in:actions', (res) => {
+    //     actions = res
+    //     resolve(null)
+    //   })
+    // })
   }
 
   isBrowser (browserName) {
@@ -398,9 +407,17 @@ export class EventManager {
     return this.Cypress.isBrowser(browserName)
   }
 
-  initialize ($autIframe: JQuery<HTMLIFrameElement>, config: Record<string, any>) {
+  async initialize ($autIframe: JQuery<HTMLIFrameElement>, config: Record<string, any>) {
     performance.mark('initialize-start')
 
+    const actions = await Cypress.backend('burn:in:actions')
+
+    window.burnInActions = actions
+
+    console.log('==========================================================================================================================================================================================================================================')
+    console.log(JSON.stringify(actions, null, 2))
+    // console.log('==========================================================================================================================================================================================================================================')
+
     const testFilter = this.specStore.testFilter
 
     return Cypress.initialize({

packages/app/src/runner/index.ts

@@ -231,7 +231,7 @@ export function addCrossOriginIframe (location) {
  * Cypress on it.
  *
  */
-function runSpecCT (config, spec: SpecFile) {
+async function runSpecCT (config, spec: SpecFile) {
   const $runnerRoot = getRunnerElement()
 
   // clear AUT, if there is one.
@@ -261,7 +261,7 @@ function runSpecCT (config, spec: SpecFile) {
   $autIframe.prop('src', specSrc)
 
   // initialize Cypress (driver) with the AUT!
-  getEventManager().initialize($autIframe, config)
+  await getEventManager().initialize($autIframe, config)
 }
 
 /**
@@ -292,7 +292,7 @@ function setSpecForDriver (spec: SpecFile) {
  * a Spec IFrame to load the spec's source code, and
  * initialize Cypress on the AUT.
  */
-function runSpecE2E (config, spec: SpecFile) {
+async function runSpecE2E (config, spec: SpecFile) {
   const $runnerRoot = getRunnerElement()
 
   // clear AUT, if there is one.
@@ -340,7 +340,7 @@ function runSpecE2E (config, spec: SpecFile) {
   })
 
   // initialize Cypress (driver) with the AUT!
-  getEventManager().initialize($autIframe, config)
+  await getEventManager().initialize($autIframe, config)
 }
 
 export function getRunnerConfigFromWindow () {
@@ -433,11 +433,11 @@ async function executeSpec (spec: SpecFile, isRerun: boolean = false) {
   await getEventManager().setup(config)
 
   if (window.__CYPRESS_TESTING_TYPE__ === 'e2e') {
-    return runSpecE2E(config, spec)
+    return await runSpecE2E(config, spec)
   }
 
   if (window.__CYPRESS_TESTING_TYPE__ === 'component') {
-    return runSpecCT(config, spec)
+    return await runSpecCT(config, spec)
   }
 
   throw Error('Unknown or undefined testingType on window.__CYPRESS_TESTING_TYPE__')

packages/driver/src/cypress/mocha.ts

@@ -1,3 +1,4 @@
+console.log('root 2')
 /* eslint-disable prefer-rest-params */
 import _ from 'lodash'
 import $errUtils, { CypressError } from './error_utils'
@@ -37,10 +38,29 @@ delete (window as any).Mocha
 
 export const SKIPPED_DUE_TO_BROWSER_MESSAGE = ' (skipped due to browser)'
 
+type LatestScore = null | -2 | -1 | 0 | 1
+
+type AttemptStrategy = 'RETRY' | 'BURN_IN' | 'NONE'
+
+type ReasonToStop =
+| 'PASSED_FIRST_ATTEMPT' // no burn-in needed
+| 'PASSED_BURN_IN' // achieved burn-in
+| 'PASSED_MET_THRESHOLD' // passed after reaching threshold for strategy 'detect-flake-and-pass-on-threshold'
+| 'FAILED_NO_RETRIES' // failed and no retries
+| 'FAILED_REACHED_MAX_RETRIES' // failed after reaching max retries
+| 'FAILED_DID_NOT_MEET_THRESHOLD' // failed since it's impossible to meet threshold for strategy 'detect-flake-and-pass-on-threshold'
+| 'FAILED_STOPPED_ON_FLAKE' // failed with one attempt passing and using strategy 'detect-flake-but-always-fail' with `stopIfAnyPassed` set to true
+// NOTE: can we detect this? how? the goal is to avoid retrying a test that failed because of a hook failure
+| 'FAILED_HOOK_FAILED' // failed because a hook failed
+
 interface CypressTest extends Mocha.Test {
   prevAttempts: CypressTest[]
   final?: boolean
   forceState?: 'passed'
+  latestScore?: LatestScore
+  thisAttemptInitialStrategy?: AttemptStrategy
+  nextAttemptStrategy?: AttemptStrategy
+  reasonToStop?: ReasonToStop
 }
 
 type Strategy = 'detect-flake-and-pass-on-threshold' | 'detect-flake-but-always-fail' | undefined
@@ -56,8 +76,31 @@ type Options<T> = T extends 'detect-flake-and-pass-on-threshold' ?
   } :
     undefined
 
+type CompleteBurnInConfig = {
+  enabled: boolean
+  default: number
+  flaky: number
+}
+
+function getNeededBurnInAttempts (latestScore: LatestScore, burnInConfig: CompleteBurnInConfig) {
+  if (burnInConfig.enabled === false) {
+    return 0
+  }
+
+  switch (latestScore) {
+    case null: return burnInConfig.default // this means the cloud determined the test is new or modified
+    case 0: return burnInConfig.default // this means the cloud determined the test was failing with no flake
+    case -1: return burnInConfig.flaky // this means the cloud determined the test was flaky
+    case -2: return 0 // this means the cloud couldn't determine the score
+    case 1: return 0 // this means the cloud determined the test graduated burn-in
+    default: return 0
+  }
+}
+
 // NOTE: 'calculateTestStatus' is marked as an individual function to make functionality easier to test.
-export function calculateTestStatus (test: CypressTest, strategy: Strategy, options: Options<Strategy>) {
+export function calculateTestStatus (test: CypressTest, strategy: Strategy, options: Options<Strategy>, completeBurnInConfig: CompleteBurnInConfig, latestScore: 0 | 1 | -1 | -2 | null) {
+  const neededBurnInAttempts = 0 //getNeededBurnInAttempts(latestScore, completeBurnInConfig)
+
   // @ts-expect-error
   const totalAttemptsAlreadyExecuted = test.currentRetry() + 1
   let shouldAttemptsContinue: boolean = true
@@ -73,10 +116,17 @@ export function calculateTestStatus (test: CypressTest, strategy: Strategy, opti
     failedTests.push(test)
   }
 
+  if (!test.prevAttempts?.length) {
+    test.thisAttemptInitialStrategy = 'NONE'
+  } else {
+    test.thisAttemptInitialStrategy = test.prevAttempts[test.prevAttempts.length - 1].nextAttemptStrategy
+  }
+
   // If there is AT LEAST one failed test attempt, we know we need to apply retry logic.
   // Otherwise, the test might be burning in (not implemented yet) OR the test passed on the first attempt,
   // meaning retry logic does NOT need to be applied.
   if (failedTests.length > 0) {
+    test.nextAttemptStrategy = 'RETRY'
     const maxAttempts = test.retries() + 1
     const remainingAttempts = maxAttempts - totalAttemptsAlreadyExecuted
     const passingAttempts = passedTests.length
@@ -95,40 +145,68 @@ export function calculateTestStatus (test: CypressTest, strategy: Strategy, opti
       ((options as Options<'detect-flake-but-always-fail'>).stopIfAnyPassed || false) :
       null
 
-    // Do we have the required amount of passes? If yes, we no longer need to keep running the test.
-    if (strategy !== 'detect-flake-but-always-fail' && passingAttempts >= (passesRequired as number)) {
-      outerTestStatus = 'passed'
-      test.final = true
-      shouldAttemptsContinue = false
-    } else if (totalAttemptsAlreadyExecuted < maxAttempts &&
-      (
-        // For strategy "detect-flake-and-pass-on-threshold" or no strategy (current GA retries):
-        //  If we haven't met our max attempt limit AND we have enough remaining attempts that can satisfy the passing requirement.
-        // retry the test.
-        (strategy !== 'detect-flake-but-always-fail' && remainingAttempts >= (neededPassingAttemptsLeft as number)) ||
-        // For strategy "detect-flake-but-always-fail":
-        //  If we haven't met our max attempt limit AND
-        //    stopIfAnyPassed is false OR
-        //    stopIfAnyPassed is true and no tests have passed yet.
-        // retry the test.
-        (strategy === 'detect-flake-but-always-fail' && (!stopIfAnyPassed || stopIfAnyPassed && passingAttempts === 0))
-      )) {
-      test.final = false
+    switch (strategy) {
+      case 'detect-flake-and-pass-on-threshold':
+        if (passingAttempts >= (passesRequired as number)) {
+          // we met the threshold, so we can stop retrying and pass the test
+          outerTestStatus = 'passed'
+          test.final = true
+          shouldAttemptsContinue = false
+          test.reasonToStop = 'PASSED_MET_THRESHOLD'
+        } else if (remainingAttempts < (neededPassingAttemptsLeft as number)) {
+          // we don't have enough remaining attempts to meet the threshold, so we should stop retrying and fail the test
+          outerTestStatus = 'failed'
+          test.final = true
+          test.forceState = test.state === 'passed' ? test.state : undefined
+          shouldAttemptsContinue = false
+          test.reasonToStop = 'FAILED_DID_NOT_MEET_THRESHOLD'
+        } else {
+          // we haven't met the threshold, but we have enough remaining attempts to meet the threshold, so we should retry the test
+          test.final = false
+          shouldAttemptsContinue = true
+        }
+
+        break
+      case 'detect-flake-but-always-fail':
+        if (stopIfAnyPassed && passingAttempts > 0) {
+          // we have a passing attempt and we should stop retrying and fail the test
+          outerTestStatus = 'failed'
+          test.final = true
+          test.forceState = test.state === 'passed' ? test.state : undefined
+          shouldAttemptsContinue = false
+          test.reasonToStop = 'FAILED_STOPPED_ON_FLAKE'
+        } else if (remainingAttempts === 0) {
+          // we have no remaining attempts and we should stop retrying and fail the test
+          outerTestStatus = 'failed'
+          test.final = true
+          test.forceState = test.state === 'passed' ? test.state : undefined
+          shouldAttemptsContinue = false
+          test.reasonToStop = 'FAILED_REACHED_MAX_RETRIES'
+        } else {
+          // we have remaining attempts and we should retry the test
+          test.final = false
+          shouldAttemptsContinue = true
+        }
+
+        break
+      default:
+        outerTestStatus = 'failed'
+        test.final = true
+        test.forceState = test.state === 'passed' ? test.state : undefined
+        shouldAttemptsContinue = false
+        test.reasonToStop = 'FAILED_NO_RETRIES'
+    }
+  } else {
+    test.nextAttemptStrategy = 'BURN_IN'
+    if (neededBurnInAttempts > passedTests.length) {
       shouldAttemptsContinue = true
+      test.final = false
     } else {
-      // Otherwise, we should stop retrying the test.
-      outerTestStatus = 'failed'
-      test.final = true
-      // If an outerStatus is 'failed', but the last test attempt was 'passed', we need to force the status so mocha doesn't flag the test attempt as failed.
-      // This is a common use case with 'detect-flake-but-always-fail', where we want to display the last attempt as 'passed' but fail the test.
-      test.forceState = test.state === 'passed' ? test.state : undefined
+      test.reasonToStop = neededBurnInAttempts > 0 ? 'PASSED_BURN_IN' : 'PASSED_FIRST_ATTEMPT'
+      outerTestStatus = 'passed'
       shouldAttemptsContinue = false
+      test.final = true
     }
-  } else {
-    // retry logic did not need to be applied and the test passed.
-    outerTestStatus = 'passed'
-    shouldAttemptsContinue = false
-    test.final = true
   }
 
   return {
@@ -438,16 +516,38 @@ function patchTestClone () {
   }
 }
 
-function createCalculateTestStatus (Cypress: Cypress.Cypress) {
+function createCalculateTestStatus (cy: Cypress.Cypress) {
   // Adds a method to the test object called 'calculateTestStatus'
   // which is used inside our mocha patch (./driver/patches/mocha+7.0.1.dev.patch)
   // in order to calculate test retries. This prototype functions as a light abstraction around
   // 'calculateTestStatus', which makes the function easier to unit-test
   Test.prototype.calculateTestStatus = function () {
-    let retriesConfig = Cypress.config('retries')
+    let retriesConfig = cy.config('retries')
+    //const testConfig = originalConfig.randomKey
+
+    const burnInActions = window.burnInActions
+
+    console.log({ burnInActions })
+    // @ts-expect-error
+    // const burnInConfig = cy.config('experimentalBurnIn')
+
+    // let burnInAction: any = null
+
+    console.log('point 1')
+    // debugger
+
+    // @ts-expect-error
+    const burnInAction = cy.backend('burnIn:action', this.id, burnInConfig)
+
+    console.log('point 2')
+    // debugger
+
+    // const startingScore = burnInAction.startingScore
+
+    // const completeBurnInConfig = burnInAction.completeBurnInConfig
 
     // @ts-expect-error
-    return calculateTestStatus(this, retriesConfig?.experimentalStrategy, retriesConfig?.experimentalOptions)
+    return calculateTestStatus(this, retriesConfig?.experimentalStrategy, retriesConfig?.experimentalOptions, { default: 3, flaky: 5, enabled: true }, -1)
   }
 }
 

packages/server/lib/modes/record.js

@@ -1084,6 +1084,13 @@ const createRunAndRecordSpecs = (options = {}) => {
           return
         }
 
+        const burnInActions = _.chain(response.actions).filter({ action: 'BURN_IN' }).map((x) => _.omit(x, ['action', 'type'])).value()
+
+        if (_.some(response.actions, { action: 'BURN_IN' })) {
+          debug(`received burn in actions %o`, burnInActions)
+          project.setBurnInActions(burnInActions)
+        }
+
         return cb(response)
       })
 

packages/server/lib/project-base.ts

@@ -18,7 +18,7 @@ import { SocketE2E } from './socket-e2e'
 import { ensureProp } from './util/class-helpers'
 
 import system from './util/system'
-import type { BannersState, FoundBrowser, FoundSpec, OpenProjectLaunchOptions, ReceivedCypressOptions, ResolvedConfigurationOptions, TestingType, VideoRecording } from '@packages/types'
+import type { BannersState, FoundBrowser, FoundSpec, OpenProjectLaunchOptions, ReceivedCypressOptions, ResolvedConfigurationOptions, TestingType, VideoRecording, BurnInAction } from '@packages/types'
 import { DataContext, getCtx } from '@packages/data-context'
 import { createHmac } from 'crypto'
 import type ProtocolManager from './cloud/protocol'
@@ -44,6 +44,7 @@ export interface Cfg extends ReceivedCypressOptions {
   component: Partial<Cfg>
   additionalIgnorePattern?: string | string[]
   resolved: ResolvedConfigurationOptions
+  randomKey: string[]
 }
 
 const localCwd = process.cwd()
@@ -70,6 +71,7 @@ export class ProjectBase extends EE {
   public testingType: Cypress.TestingType
   public spec: FoundSpec | null
   public isOpen: boolean = false
+  private _burnInActions: BurnInAction[] = []
   projectRoot: string
 
   constructor ({
@@ -430,6 +432,10 @@ export class ProjectBase extends EE {
     this.browser = browser
   }
 
+  setBurnInActions (actions: BurnInAction[]) {
+    this._server?.setBurnInActions(actions)
+  }
+
   get protocolManager (): ProtocolManager | undefined {
     return this._protocolManager
   }

packages/server/lib/server-base.ts

@@ -28,7 +28,7 @@ import { createInitialWorkers } from '@packages/rewriter'
 import type { Cfg } from './project-base'
 import type { Browser } from '@packages/server/lib/browsers/types'
 import { InitializeRoutes, createCommonRoutes } from './routes'
-import type { FoundSpec, ProtocolManagerShape, TestingType } from '@packages/types'
+import type { BurnInAction, FoundSpec, ProtocolManagerShape, TestingType } from '@packages/types'
 import type { Server as WebSocketServer } from 'ws'
 import { RemoteStates } from './remote_states'
 import { cookieJar, SerializableAutomationCookie } from './util/cookies'
@@ -218,6 +218,10 @@ export class ServerBase<TSocket extends SocketE2E | SocketCt> {
     this._networkProxy?.setProtocolManager(protocolManager)
   }
 
+  setBurnInActions (actions: BurnInAction[]) {
+    this._socket?.setBurnInActions(actions)
+  }
+
   setupCrossOriginRequestHandling () {
     this._eventBus.on('cross:origin:cookies', (cookies: SerializableAutomationCookie[]) => {
       this.socket.localBus.once('cross:origin:cookies:received', () => {