chore: add cy-in-cy tests to verify reporter behavior for pass/fail t…

…ests, as well as new mocha snapshots to verify attempts. New tests were needed for this as the 'retries' option in testConfigOverrides currently is and will be invalid for experiment and will function as an override. tests run in the cy-in-cy tests are using globally configured experimentalRetries for the given tested project, which showcases the different behavior between attempts/retries and pass/fail status.

packages/app/cypress/e2e/runner/runner.experimentalRetries.mochaEvents.cy.ts

@@ -0,0 +1,173 @@
+import { runSpec } from './support/spec-loader'
+import { runCypressInCypressMochaEventsTest } from './support/mochaEventsUtils'
+import { snapshots } from './runner.experimentalRetries.mochaEvents.snapshots'
+
+/**
+ * The mochaEvent tests require a spec to be loaded and executed within an inner Cypress context.
+ * The spec must load and execute within the duration of the Cypress command timeout.
+ * The execution time of the inner Cypress is resource/OS dependant and can exceed the default value (4s),
+ * so we have increased the command timeout to allow the inner spec more time to complete and report
+ * its mocha event log.
+ */
+
+/**
+ * In context to experimentalRetries, the end state of the tests should be identical regardless of strategy for these tests.
+ * However, the total amount of attempts on a test will differ based on the strategy used and is documented below
+ */
+describe('experimental retries: runner tests', { defaultCommandTimeout: 7500 }, () => {
+  const projects: ['detect-flake-and-pass-on-threshold', 'detect-flake-but-always-fail', 'detect-flake-but-always-fail-stop-any-passed'] = ['detect-flake-and-pass-on-threshold', 'detect-flake-but-always-fail', 'detect-flake-but-always-fail-stop-any-passed']
+
+  projects.forEach((project) => {
+    describe(project, () => {
+      describe('tests finish with correct state', () => {
+        describe('hook failures', () => {
+          // regardless of strategy, this should fail the suite immediately and not run any additional attempts, so the snapshots should be near identical
+          it(`fail in [before]`, (done) => {
+            const { assertMatchingSnapshot } = runCypressInCypressMochaEventsTest(
+              snapshots,
+              `"${project}": tests finish with correct state hook failures fail in [before] #1`,
+              done,
+            )
+
+            runSpec({
+              fileName: 'fail-with-before.mochaEvents.cy.js',
+              projectName: project,
+            }).then((win) => {
+              assertMatchingSnapshot(win)
+            })
+          })
+
+          // This will differ per strategy
+          // the snapshots for 'detect-flake-and-always-fail' configurations should almost be identical, regardless of experimentalOptions configuration.
+          // for each project:
+          // 'detect-flake-and-pass-on-threshold': will run a total of 6 times and fail 6 times, config is satisfied, the test fails, and the suite is skipped
+          // 'detect-flake-but-always-fail': will run a total of 10 times and fail 10 times, config is satisfied, the test fails, and the suite is skipped
+          // 'detect-flake-but-always-fail-stop-any-passed': will run a total of 10 times and fail 10 times config is satisfied, the test fails, and the suite is skipped
+          it(`fail in [beforeEach]`, (done) => {
+            const { assertMatchingSnapshot } = runCypressInCypressMochaEventsTest(
+              snapshots,
+              `"${project}": tests finish with correct state hook failures fail in [beforeEach] #1`,
+              done,
+            )
+
+            runSpec({
+              fileName: 'fail-with-beforeEach.mochaEvents.cy.js',
+              projectName: project,
+            }).then((win) => {
+              assertMatchingSnapshot(win)
+            })
+          })
+
+          // regardless of strategy, this should fail the suite immediately and not run any additional attempts, so the snapshots should be near identical
+          it(`fail in [after]`, (done) => {
+            const { assertMatchingSnapshot } = runCypressInCypressMochaEventsTest(
+              snapshots,
+              `"${project}": tests finish with correct state hook failures fail in [after] #1`,
+              done,
+            )
+
+            runSpec({
+              fileName: 'fail-with-after.mochaEvents.cy.js',
+              projectName: project,
+            }).then((win) => {
+              assertMatchingSnapshot(win)
+            })
+          })
+
+          // This will differ per strategy
+          // the snapshots for 'detect-flake-and-always-fail' configurations should almost be identical, regardless of experimentalOptions configuration.
+          // for each project:
+          // 'detect-flake-and-pass-on-threshold': will run a total of 6 times and fail 6 times, config is satisfied, the test fails, and the suite is skipped
+          // 'detect-flake-but-always-fail': will run a total of 10 times and fail 10 times, config is satisfied, the test fails, and the suite is skipped
+          // 'detect-flake-but-always-fail-stop-any-passed': will run a total of 10 times and fail 10 times config is satisfied, the test fails, and the suite is skipped
+          it(`fail in [afterEach]`, (done) => {
+            const { assertMatchingSnapshot } = runCypressInCypressMochaEventsTest(
+              snapshots,
+              `"${project}": tests finish with correct state hook failures fail in [afterEach] #1`,
+              done,
+            )
+
+            runSpec({
+              fileName: 'fail-with-afterEach.mochaEvents.cy.js',
+              projectName: project,
+            }).then((win) => {
+              assertMatchingSnapshot(win)
+            })
+          })
+        })
+      })
+
+      describe('mocha grep', () => {
+        // This will differ per strategy
+        // the snapshots for 'detect-flake-and-always-fail' configurations should almost be identical, regardless of experimentalOptions configuration.
+        // for each project:
+        // 'detect-flake-and-pass-on-threshold': will run a total of 6 times and fail 6 times, config is satisfied, the test fails, but the suite is NOT skipped
+        // 'detect-flake-but-always-fail': will run a total of 10 times and fail 10 times, config is satisfied, the test fails, but the suite is NOT skipped
+        // 'detect-flake-but-always-fail-stop-any-passed': will run a total of 10 times and fail 10 times config is satisfied, the test fails,but the suite is NOT skipped
+        it('fail with [only]', (done) => {
+          const { assertMatchingSnapshot } = runCypressInCypressMochaEventsTest(
+            snapshots,
+            `"${project}": tests finish with correct state mocha grep fail with [only] #1`,
+            done,
+          )
+
+          runSpec({
+            fileName: 'fail-with-only.mochaEvents.cy.js',
+            projectName: project,
+          }).then((win) => {
+            assertMatchingSnapshot(win)
+          })
+        })
+
+        // This will be the same per strategy, as the test passes and retries don't get invoked
+        it('pass with [only]', (done) => {
+          const { assertMatchingSnapshot } = runCypressInCypressMochaEventsTest(
+            snapshots,
+            `"${project}": tests finish with correct state mocha grep pass with [only] #1`,
+            done,
+          )
+
+          runSpec({
+            fileName: 'pass-with-only.mochaEvents.cy.js',
+            projectName: project,
+          }).then((win) => {
+            assertMatchingSnapshot(win)
+          })
+        })
+      })
+    })
+
+    // these should be the same per strategy, as each test passes and retries is not invoked
+    describe('mocha events', () => {
+      it('simple single test', (done) => {
+        const { assertMatchingSnapshot } = runCypressInCypressMochaEventsTest(
+          snapshots,
+          `"${project}": mocha events simple single test #1`,
+          done,
+        )
+
+        runSpec({
+          fileName: 'simple-single-test.mochaEvents.cy.js',
+          projectName: project,
+        }).then((win) => {
+          assertMatchingSnapshot(win)
+        })
+      })
+
+      it('simple three tests', (done) => {
+        const { assertMatchingSnapshot } = runCypressInCypressMochaEventsTest(
+          snapshots,
+          `"${project}": mocha events simple three tests #1`,
+          done,
+        )
+
+        runSpec({
+          fileName: 'three-tests-with-hooks.mochaEvents.cy.js',
+          projectName: project,
+        }).then((win) => {
+          assertMatchingSnapshot(win)
+        })
+      })
+    })
+  })
+})