diff --git a/packages/app/cypress/e2e/runner/retries.experimentalRetries.mochaEvents.snapshots.ts b/packages/app/cypress/e2e/runner/retries.experimentalRetries.mochaEvents.snapshots.ts index 0d29be5d5b5e..c4dcf888d418 100644 --- a/packages/app/cypress/e2e/runner/retries.experimentalRetries.mochaEvents.snapshots.ts +++ b/packages/app/cypress/e2e/runner/retries.experimentalRetries.mochaEvents.snapshots.ts @@ -40285,6 +40285,12 @@ export const snapshots = { 'mocha', 'fail', { + _cypressTestStatusInfo: { + attempts: 1, + strategy: 'detect-flake-and-pass-on-threshold', + outerStatus: 'failed', + shouldAttemptsContinue: false, + }, id: 'r3', title: '"before all" hook for "test 1"', hookName: 'before all', @@ -40612,6 +40618,12 @@ export const snapshots = { 'mocha', 'fail', { + _cypressTestStatusInfo: { + attempts: 1, + strategy: 'detect-flake-but-always-fail', + outerStatus: 'failed', + shouldAttemptsContinue: false, + }, id: 'r3', title: '"before all" hook for "test 1"', hookName: 'before all', @@ -40940,6 +40952,12 @@ export const snapshots = { 'mocha', 'fail', { + _cypressTestStatusInfo: { + attempts: 1, + strategy: 'detect-flake-but-always-fail', + outerStatus: 'failed', + shouldAttemptsContinue: false, + }, id: 'r3', title: '"before all" hook for "test 1"', hookName: 'before all', diff --git a/packages/app/cypress/e2e/runner/retries.mochaEvents.snapshots.ts b/packages/app/cypress/e2e/runner/retries.mochaEvents.snapshots.ts index 5e4e97e95807..f3ddef3c2dfa 100644 --- a/packages/app/cypress/e2e/runner/retries.mochaEvents.snapshots.ts +++ b/packages/app/cypress/e2e/runner/retries.mochaEvents.snapshots.ts @@ -6327,6 +6327,11 @@ export const snapshots = { 'mocha', 'fail', { + _cypressTestStatusInfo: { + attempts: 1, + outerStatus: 'failed', + shouldAttemptsContinue: false, + }, id: 'r3', title: '"before all" hook for "test 1"', hookName: 'before all', diff --git a/packages/app/cypress/e2e/runner/runner.experimentalRetries.mochaEvents.snapshots.ts b/packages/app/cypress/e2e/runner/runner.experimentalRetries.mochaEvents.snapshots.ts index 93a1a226cf31..c30468d930f3 100644 --- a/packages/app/cypress/e2e/runner/runner.experimentalRetries.mochaEvents.snapshots.ts +++ b/packages/app/cypress/e2e/runner/runner.experimentalRetries.mochaEvents.snapshots.ts @@ -89,6 +89,12 @@ export const snapshots = { 'mocha', 'fail', { + _cypressTestStatusInfo: { + attempts: 1, + strategy: 'detect-flake-and-pass-on-threshold', + outerStatus: 'failed', + shouldAttemptsContinue: false, + }, id: 'r3', title: '"before all" hook for "test 1"', hookName: 'before all', @@ -358,6 +364,12 @@ export const snapshots = { 'mocha', 'fail', { + _cypressTestStatusInfo: { + attempts: 1, + strategy: 'detect-flake-but-always-fail', + outerStatus: 'failed', + shouldAttemptsContinue: false, + }, id: 'r3', title: '"before all" hook for "test 1"', hookName: 'before all', @@ -627,6 +639,12 @@ export const snapshots = { 'mocha', 'fail', { + _cypressTestStatusInfo: { + attempts: 1, + strategy: 'detect-flake-but-always-fail', + outerStatus: 'failed', + shouldAttemptsContinue: false, + }, id: 'r3', title: '"before all" hook for "test 1"', hookName: 'before all', @@ -2167,6 +2185,12 @@ export const snapshots = { 'mocha', 'fail', { + _cypressTestStatusInfo: { + attempts: 6, + strategy: 'detect-flake-and-pass-on-threshold', + outerStatus: 'failed', + shouldAttemptsContinue: false, + }, id: 'r3', title: '"before each" hook for "test 1"', hookName: 'before each', @@ -4742,6 +4766,12 @@ export const snapshots = { 'mocha', 'fail', { + _cypressTestStatusInfo: { + attempts: 10, + strategy: 'detect-flake-but-always-fail', + outerStatus: 'failed', + shouldAttemptsContinue: false, + }, id: 'r3', title: '"before each" hook for "test 1"', hookName: 'before each', @@ -7317,6 +7347,12 @@ export const snapshots = { 'mocha', 'fail', { + _cypressTestStatusInfo: { + attempts: 10, + strategy: 'detect-flake-but-always-fail', + outerStatus: 'failed', + shouldAttemptsContinue: false, + }, id: 'r3', title: '"before each" hook for "test 1"', hookName: 'before each', @@ -7863,6 +7899,12 @@ export const snapshots = { 'mocha', 'fail', { + _cypressTestStatusInfo: { + attempts: 1, + strategy: 'detect-flake-and-pass-on-threshold', + outerStatus: 'failed', + shouldAttemptsContinue: false, + }, id: 'r4', title: '"after all" hook for "test 2"', hookName: 'after all', @@ -8404,6 +8446,12 @@ export const snapshots = { 'mocha', 'fail', { + _cypressTestStatusInfo: { + attempts: 1, + strategy: 'detect-flake-but-always-fail', + outerStatus: 'failed', + shouldAttemptsContinue: false, + }, id: 'r4', title: '"after all" hook for "test 2"', hookName: 'after all', @@ -8945,6 +8993,12 @@ export const snapshots = { 'mocha', 'fail', { + _cypressTestStatusInfo: { + attempts: 1, + strategy: 'detect-flake-but-always-fail', + outerStatus: 'failed', + shouldAttemptsContinue: false, + }, id: 'r4', title: '"after all" hook for "test 2"', hookName: 'after all', @@ -10780,6 +10834,12 @@ export const snapshots = { 'mocha', 'fail', { + _cypressTestStatusInfo: { + attempts: 6, + strategy: 'detect-flake-and-pass-on-threshold', + outerStatus: 'failed', + shouldAttemptsContinue: false, + }, id: 'r3', title: '"after each" hook for "test 1"', hookName: 'after each', @@ -13807,6 +13867,12 @@ export const snapshots = { 'mocha', 'fail', { + _cypressTestStatusInfo: { + attempts: 10, + strategy: 'detect-flake-but-always-fail', + outerStatus: 'failed', + shouldAttemptsContinue: false, + }, id: 'r3', title: '"after each" hook for "test 1"', hookName: 'after each', @@ -16530,6 +16596,12 @@ export const snapshots = { 'mocha', 'fail', { + _cypressTestStatusInfo: { + attempts: 10, + strategy: 'detect-flake-but-always-fail', + outerStatus: 'failed', + shouldAttemptsContinue: false, + }, id: 'r3', title: '"after each" hook for "test 1"', hookName: 'after each', diff --git a/packages/app/cypress/e2e/runner/runner.mochaEvents.snapshots.ts b/packages/app/cypress/e2e/runner/runner.mochaEvents.snapshots.ts index 2e2f15282991..2250e4d05e53 100644 --- a/packages/app/cypress/e2e/runner/runner.mochaEvents.snapshots.ts +++ b/packages/app/cypress/e2e/runner/runner.mochaEvents.snapshots.ts @@ -1,13 +1,7 @@ export const snapshots = { 'src/cypress/runner tests finish with correct state hook failures fail in [before] #1': [ - [ - 'mocha', - 'start', - { - start: 'match.date', - }, - ], + ['mocha', 'start', { start: 'match.date' }], [ 'mocha', 'suite', @@ -95,6 +89,11 @@ export const snapshots = { 'mocha', 'fail', { + _cypressTestStatusInfo: { + attempts: 1, + outerStatus: 'failed', + shouldAttemptsContinue: false, + }, id: 'r3', title: '"before all" hook for "test 1"', hookName: 'before all', @@ -272,13 +271,7 @@ export const snapshots = { _slow: 10000, }, ], - [ - 'mocha', - 'end', - { - end: 'match.date', - }, - ], + ['mocha', 'end', { end: 'match.date' }], ], 'src/cypress/runner tests finish with correct state hook failures fail in [beforeEach] #1': @@ -388,6 +381,11 @@ export const snapshots = { 'mocha', 'fail', { + _cypressTestStatusInfo: { + attempts: 1, + outerStatus: 'failed', + shouldAttemptsContinue: false, + }, id: 'r3', title: '"before each" hook for "test 1"', hookName: 'before each', @@ -609,13 +607,7 @@ export const snapshots = { 'src/cypress/runner tests finish with correct state hook failures fail in [after] #1': [ - [ - 'mocha', - 'start', - { - start: 'match.date', - }, - ], + ['mocha', 'start', { start: 'match.date' }], [ 'mocha', 'suite', @@ -940,6 +932,11 @@ export const snapshots = { 'mocha', 'fail', { + _cypressTestStatusInfo: { + attempts: 1, + outerStatus: 'failed', + shouldAttemptsContinue: false, + }, id: 'r4', title: '"after all" hook for "test 2"', hookName: 'after all', @@ -1144,13 +1141,7 @@ export const snapshots = { _slow: 10000, }, ], - [ - 'mocha', - 'end', - { - end: 'match.date', - }, - ], + ['mocha', 'end', { end: 'match.date' }], ], 'src/cypress/runner tests finish with correct state hook failures fail in [afterEach] #1': @@ -1260,6 +1251,11 @@ export const snapshots = { 'mocha', 'fail', { + _cypressTestStatusInfo: { + attempts: 1, + outerStatus: 'failed', + shouldAttemptsContinue: false, + }, id: 'r3', title: '"after each" hook for "test 1"', hookName: 'after each', diff --git a/packages/driver/src/cypress/runner.ts b/packages/driver/src/cypress/runner.ts index 6dddb4275bd3..19a4d6582054 100644 --- a/packages/driver/src/cypress/runner.ts +++ b/packages/driver/src/cypress/runner.ts @@ -1157,6 +1157,20 @@ const _runnerListeners = (_runner, Cypress, _emissions, getTestById, getTest, se } else { err = $errUtils.appendErrMsg(err, errMessage) } + + // If the test never failed and only the hooks did, + // we need to attach the metadata of the test to the hook to report the failure correctly to the server reporter. + // We calculate it fresh here since it may not be available on the test, which is the case with a beforeEach hook. + // as well as maybe incorrect (test passed on first attempt, but after hooks failed) + const testStatus = test.calculateTestStatus() + + runnable._cypressTestStatusInfo = { + attempts: testStatus.attempts, + strategy: testStatus.strategy, + // regardless of the test state, we should ultimately fail the test here. + outerStatus: runnable.state, + shouldAttemptsContinue: false, + } } // always set runnable err so we can tap into diff --git a/system-tests/__snapshots__/experimental_retries.spec.ts.js b/system-tests/__snapshots__/experimental_retries.spec.ts.js index dd93c0f172a7..16087f1740e7 100644 --- a/system-tests/__snapshots__/experimental_retries.spec.ts.js +++ b/system-tests/__snapshots__/experimental_retries.spec.ts.js @@ -1672,3 +1672,631 @@ Error: test 2 ` + +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-but-always-fail" / exercises experimental-retries hook failures to verify console reporter and final status code are correct.'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 4 found (fail-with-afterEach.mochaEvents.cy.js, fail-with-beforeEach.mochaEvents.c │ + │ y.js, fail-with-after.mochaEvents.cy.js, fail-with-before.mochaEvents.cy.js) │ + │ Searched: cypress/e2e/runner/fail-with-afterEach.mochaEvents.cy.js, cypress/e2e/runner/fail- │ + │ with-beforeEach.mochaEvents.cy.js, cypress/e2e/runner/fail-with-after.mochaEvents. │ + │ cy.js, cypress/e2e/runner/fail-with-before.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: fail-with-afterEach.mochaEvents.cy.js (1 of 4) + + + suite 1 + ✖(Attempt 1 of 10) test 1 + ✖(Attempt 2 of 10) test 1 + ✖(Attempt 3 of 10) test 1 + ✖(Attempt 4 of 10) test 1 + ✖(Attempt 5 of 10) test 1 + ✖(Attempt 6 of 10) test 1 + ✖(Attempt 7 of 10) test 1 + ✖(Attempt 8 of 10) test 1 + ✖(Attempt 9 of 10) test 1 + ✖(Attempt 10 of 10) "after each" hook for "test 1" + ✖ "after each" hook for "test 1" + + + 0 passing + 1 failing + + 1) suite 1 + "after each" hook for "test 1": + Error: After each error + +Because this error occurred during a \`after each\` hook we are skipping the remaining tests in the current suite: \`suite 1\` + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 1 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: fail-with-afterEach.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: fail-with-beforeEach.mochaEvents.cy.js (2 of 4) + + + suite 1 + ✖(Attempt 1 of 10) test 1 + ✖(Attempt 2 of 10) test 1 + ✖(Attempt 3 of 10) test 1 + ✖(Attempt 4 of 10) test 1 + ✖(Attempt 5 of 10) test 1 + ✖(Attempt 6 of 10) test 1 + ✖(Attempt 7 of 10) test 1 + ✖(Attempt 8 of 10) test 1 + ✖(Attempt 9 of 10) test 1 + ✖(Attempt 10 of 10) "before each" hook for "test 1" + ✖ "before each" hook for "test 1" + + + 0 passing + 1 failing + + 1) suite 1 + "before each" hook for "test 1": + Error: beforeEach + +Because this error occurred during a \`before each\` hook we are skipping the remaining tests in the current suite: \`suite 1\` + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 1 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: fail-with-beforeEach.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: fail-with-after.mochaEvents.cy.js (3 of 4) + + + suite 1 + ✓ test 1 + ✖ "after all" hook for "test 2" (NaNms) + + + 1 passing + 1 failing + + 1) suite 1 + "after all" hook for "test 2": + Error: Error! + +Because this error occurred during a \`after all\` hook we are skipping the remaining tests in the current suite: \`suite 1\` + +Although you have test retries enabled, we do not retry tests when \`before all\` or \`after all\` hooks fail + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 1 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: fail-with-after.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: fail-with-before.mochaEvents.cy.js (4 of 4) + + + suite 1 + ✖ "before all" hook for "test 1" (NaNms) + + + 0 passing + 1 failing + + 1) suite 1 + "before all" hook for "test 1": + Error: before + +Because this error occurred during a \`before all\` hook we are skipping the remaining tests in the current suite: \`suite 1\` + +Although you have test retries enabled, we do not retry tests when \`before all\` or \`after all\` hooks fail + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: fail-with-before.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✖ fail-with-afterEach.mochaEvents.cy. XX:XX 2 - 1 - 1 │ + │ js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ fail-with-beforeEach.mochaEvents.cy XX:XX 2 - 1 - 1 │ + │ .js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ fail-with-after.mochaEvents.cy.js XX:XX 2 1 1 - - │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ fail-with-before.mochaEvents.cy.js XX:XX 1 - 1 - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✖ 4 of 4 failed (100%) XX:XX 7 1 4 - 2 + + +` + +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-but-always-fail" / exercises experimental-retries hook failures to verify console reporter and final status code are correct (stopIfAnyPassed=true).'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 4 found (fail-with-afterEach.mochaEvents.cy.js, fail-with-beforeEach.mochaEvents.c │ + │ y.js, fail-with-after.mochaEvents.cy.js, fail-with-before.mochaEvents.cy.js) │ + │ Searched: cypress/e2e/runner/fail-with-afterEach.mochaEvents.cy.js, cypress/e2e/runner/fail- │ + │ with-beforeEach.mochaEvents.cy.js, cypress/e2e/runner/fail-with-after.mochaEvents. │ + │ cy.js, cypress/e2e/runner/fail-with-before.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: fail-with-afterEach.mochaEvents.cy.js (1 of 4) + + + suite 1 + ✖(Attempt 1 of 10) test 1 + ✖(Attempt 2 of 10) test 1 + ✖(Attempt 3 of 10) test 1 + ✖(Attempt 4 of 10) test 1 + ✖(Attempt 5 of 10) test 1 + ✖(Attempt 6 of 10) test 1 + ✖(Attempt 7 of 10) test 1 + ✖(Attempt 8 of 10) test 1 + ✖(Attempt 9 of 10) test 1 + ✖(Attempt 10 of 10) "after each" hook for "test 1" + ✖ "after each" hook for "test 1" + + + 0 passing + 1 failing + + 1) suite 1 + "after each" hook for "test 1": + Error: After each error + +Because this error occurred during a \`after each\` hook we are skipping the remaining tests in the current suite: \`suite 1\` + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 1 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: fail-with-afterEach.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: fail-with-beforeEach.mochaEvents.cy.js (2 of 4) + + + suite 1 + ✖(Attempt 1 of 10) test 1 + ✖(Attempt 2 of 10) test 1 + ✖(Attempt 3 of 10) test 1 + ✖(Attempt 4 of 10) test 1 + ✖(Attempt 5 of 10) test 1 + ✖(Attempt 6 of 10) test 1 + ✖(Attempt 7 of 10) test 1 + ✖(Attempt 8 of 10) test 1 + ✖(Attempt 9 of 10) test 1 + ✖(Attempt 10 of 10) "before each" hook for "test 1" + ✖ "before each" hook for "test 1" + + + 0 passing + 1 failing + + 1) suite 1 + "before each" hook for "test 1": + Error: beforeEach + +Because this error occurred during a \`before each\` hook we are skipping the remaining tests in the current suite: \`suite 1\` + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 1 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: fail-with-beforeEach.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: fail-with-after.mochaEvents.cy.js (3 of 4) + + + suite 1 + ✓ test 1 + ✖ "after all" hook for "test 2" (NaNms) + + + 1 passing + 1 failing + + 1) suite 1 + "after all" hook for "test 2": + Error: Error! + +Because this error occurred during a \`after all\` hook we are skipping the remaining tests in the current suite: \`suite 1\` + +Although you have test retries enabled, we do not retry tests when \`before all\` or \`after all\` hooks fail + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 1 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: fail-with-after.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: fail-with-before.mochaEvents.cy.js (4 of 4) + + + suite 1 + ✖ "before all" hook for "test 1" (NaNms) + + + 0 passing + 1 failing + + 1) suite 1 + "before all" hook for "test 1": + Error: before + +Because this error occurred during a \`before all\` hook we are skipping the remaining tests in the current suite: \`suite 1\` + +Although you have test retries enabled, we do not retry tests when \`before all\` or \`after all\` hooks fail + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: fail-with-before.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✖ fail-with-afterEach.mochaEvents.cy. XX:XX 2 - 1 - 1 │ + │ js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ fail-with-beforeEach.mochaEvents.cy XX:XX 2 - 1 - 1 │ + │ .js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ fail-with-after.mochaEvents.cy.js XX:XX 2 1 1 - - │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ fail-with-before.mochaEvents.cy.js XX:XX 1 - 1 - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✖ 4 of 4 failed (100%) XX:XX 7 1 4 - 2 + + +` + +exports['e2e retries.experimentalStrategy / experimentalBurnIn=false / "detect-flake-and-pass-on-threshold" / exercises experimental-retries hook failures to verify console reporter and final status code are correct.'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 4 found (fail-with-afterEach.mochaEvents.cy.js, fail-with-beforeEach.mochaEvents.c │ + │ y.js, fail-with-after.mochaEvents.cy.js, fail-with-before.mochaEvents.cy.js) │ + │ Searched: cypress/e2e/runner/fail-with-afterEach.mochaEvents.cy.js, cypress/e2e/runner/fail- │ + │ with-beforeEach.mochaEvents.cy.js, cypress/e2e/runner/fail-with-after.mochaEvents. │ + │ cy.js, cypress/e2e/runner/fail-with-before.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: fail-with-afterEach.mochaEvents.cy.js (1 of 4) + + + suite 1 + ✖(Attempt 1 of 10) test 1 + ✖(Attempt 2 of 10) test 1 + ✖(Attempt 3 of 10) test 1 + ✖(Attempt 4 of 10) test 1 + ✖(Attempt 5 of 10) test 1 + ✖(Attempt 6 of 10) "after each" hook for "test 1" + ✖ "after each" hook for "test 1" + + + 0 passing + 1 failing + + 1) suite 1 + "after each" hook for "test 1": + Error: After each error + +Because this error occurred during a \`after each\` hook we are skipping the remaining tests in the current suite: \`suite 1\` + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 1 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: fail-with-afterEach.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: fail-with-beforeEach.mochaEvents.cy.js (2 of 4) + + + suite 1 + ✖(Attempt 1 of 10) test 1 + ✖(Attempt 2 of 10) test 1 + ✖(Attempt 3 of 10) test 1 + ✖(Attempt 4 of 10) test 1 + ✖(Attempt 5 of 10) test 1 + ✖(Attempt 6 of 10) "before each" hook for "test 1" + ✖ "before each" hook for "test 1" + + + 0 passing + 1 failing + + 1) suite 1 + "before each" hook for "test 1": + Error: beforeEach + +Because this error occurred during a \`before each\` hook we are skipping the remaining tests in the current suite: \`suite 1\` + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 1 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: fail-with-beforeEach.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: fail-with-after.mochaEvents.cy.js (3 of 4) + + + suite 1 + ✓ test 1 + ✖ "after all" hook for "test 2" (NaNms) + + + 1 passing + 1 failing + + 1) suite 1 + "after all" hook for "test 2": + Error: Error! + +Because this error occurred during a \`after all\` hook we are skipping the remaining tests in the current suite: \`suite 1\` + +Although you have test retries enabled, we do not retry tests when \`before all\` or \`after all\` hooks fail + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 1 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: fail-with-after.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: fail-with-before.mochaEvents.cy.js (4 of 4) + + + suite 1 + ✖ "before all" hook for "test 1" (NaNms) + + + 0 passing + 1 failing + + 1) suite 1 + "before all" hook for "test 1": + Error: before + +Because this error occurred during a \`before all\` hook we are skipping the remaining tests in the current suite: \`suite 1\` + +Although you have test retries enabled, we do not retry tests when \`before all\` or \`after all\` hooks fail + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 0 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Spec Ran: fail-with-before.mochaEvents.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✖ fail-with-afterEach.mochaEvents.cy. XX:XX 2 - 1 - 1 │ + │ js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ fail-with-beforeEach.mochaEvents.cy XX:XX 2 - 1 - 1 │ + │ .js │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ fail-with-after.mochaEvents.cy.js XX:XX 2 1 1 - - │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ fail-with-before.mochaEvents.cy.js XX:XX 1 - 1 - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✖ 4 of 4 failed (100%) XX:XX 7 1 4 - 2 + + +` diff --git a/system-tests/test/experimental_retries.spec.ts b/system-tests/test/experimental_retries.spec.ts index b62daa563537..d4291fdd46f4 100644 --- a/system-tests/test/experimental_retries.spec.ts +++ b/system-tests/test/experimental_retries.spec.ts @@ -139,6 +139,27 @@ describe('e2e retries.experimentalStrategy', () => { screenshotOnRunFailure: false, }, }) + + systemTests.it('exercises experimental-retries hook failures to verify console reporter and final status code are correct.', { + project: 'detect-flake-and-pass-on-threshold', + browser: '!webkit', + spec: 'runner/fail-with-afterEach.mochaEvents.cy.js,runner/fail-with-beforeEach.mochaEvents.cy.js,runner/fail-with-after.mochaEvents.cy.js,runner/fail-with-before.mochaEvents.cy.js', + snapshot: true, + expectedExitCode: 4, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-and-pass-on-threshold', + experimentalOptions: { + maxRetries: 9, + passesRequired: 5, + }, + }, + screenshotOnRunFailure: false, + }, + }) }) describe('"detect-flake-but-always-fail"', () => { @@ -236,6 +257,27 @@ describe('e2e retries.experimentalStrategy', () => { }, }) + systemTests.it('exercises experimental-retries hook failures to verify console reporter and final status code are correct.', { + project: 'detect-flake-but-always-fail', + browser: '!webkit', + spec: 'runner/fail-with-afterEach.mochaEvents.cy.js,runner/fail-with-beforeEach.mochaEvents.cy.js,runner/fail-with-after.mochaEvents.cy.js,runner/fail-with-before.mochaEvents.cy.js', + snapshot: true, + expectedExitCode: 4, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-but-always-fail', + experimentalOptions: { + maxRetries: 9, + stopIfAnyPassed: false, + }, + }, + screenshotOnRunFailure: false, + }, + }) + systemTests.it('exercises experimental-retries suite to verify console reporter and final status code are correct (stopIfAnyPassed=true).', { project: 'detect-flake-but-always-fail-stop-any-passed', browser: '!webkit', @@ -257,6 +299,27 @@ describe('e2e retries.experimentalStrategy', () => { screenshotOnRunFailure: false, }, }) + + systemTests.it('exercises experimental-retries hook failures to verify console reporter and final status code are correct (stopIfAnyPassed=true).', { + project: 'detect-flake-but-always-fail', + browser: '!webkit', + spec: 'runner/fail-with-afterEach.mochaEvents.cy.js,runner/fail-with-beforeEach.mochaEvents.cy.js,runner/fail-with-after.mochaEvents.cy.js,runner/fail-with-before.mochaEvents.cy.js', + snapshot: true, + expectedExitCode: 4, + config: { + experimentalBurnIn: false, + retries: { + openMode: false, + runMode: true, + experimentalStrategy: 'detect-flake-but-always-fail', + experimentalOptions: { + maxRetries: 9, + stopIfAnyPassed: true, + }, + }, + screenshotOnRunFailure: false, + }, + }) }) }) })