diff --git a/cli/types/cypress.d.ts b/cli/types/cypress.d.ts index 914d03ffa658..db25220c1fb4 100644 --- a/cli/types/cypress.d.ts +++ b/cli/types/cypress.d.ts @@ -2838,6 +2838,30 @@ declare namespace Cypress { certs: PEMCert[] | PFXCert[] } + type RetryStrategyWithModeSpecs = RetryStrategy & { + openMode: boolean; // defaults to false + runMode: boolean; // defaults to true + } + + type RetryStrategy = + | RetryStrategyDetectFlakeAndPassOnThresholdType + | RetryStrategyDetectFlakeButAlwaysFailType + + interface RetryStrategyDetectFlakeAndPassOnThresholdType { + experimentalStrategy: "detect-flake-and-pass-on-threshold" + experimentalOptions?: { + maxRetries: number; // defaults to 2 if experimentalOptions is not provided, must be a whole number > 0 + passesRequired: number; // defaults to 2 if experimentalOptions is not provided, must be a whole number > 0 and <= maxRetries + } + } + + interface RetryStrategyDetectFlakeButAlwaysFailType { + experimentalStrategy: "detect-flake-but-always-fail" + experimentalOptions?: { + maxRetries: number; // defaults to 2 if experimentalOptions is not provided, must be a whole number > 0 + stopIfAnyPassed: boolean; // defaults to false if experimentalOptions is not provided + } + } interface ResolvedConfigOptions { /** * Url used as prefix for [cy.visit()](https://on.cypress.io/visit) or [cy.request()](https://on.cypress.io/request) command's url @@ -3131,7 +3155,7 @@ declare namespace Cypress { * To enable test retries only in runMode, set e.g. `{ openMode: null, runMode: 2 }` * @default null */ - retries: Nullable, openMode?: Nullable }> + retries: Nullable, openMode?: Nullable }) | RetryStrategyWithModeSpecs> /** * Enables including elements within the shadow DOM when using querying * commands (e.g. cy.get(), cy.find()). Can be set globally in cypress.config.{js,ts,mjs,cjs}, diff --git a/cli/types/tests/cypress-tests.ts b/cli/types/tests/cypress-tests.ts index dd306c4ed4d6..6bba72144c26 100644 --- a/cli/types/tests/cypress-tests.ts +++ b/cli/types/tests/cypress-tests.ts @@ -1159,6 +1159,39 @@ namespace CypressLocalStorageTests { cy.clearAllSessionStorage({ log: 'true' }) // $ExpectError } +namespace CypressRetriesSpec { + Cypress.config('retries', { + openMode: 0, + runMode: 1 + }) + + Cypress.config('retries', { + openMode: false, + runMode: false, + experimentalStrategy: "detect-flake-and-pass-on-threshold", + experimentalOptions: { + maxRetries: 2, + passesRequired: 2 + } + }) + + Cypress.config('retries', { + openMode: false, + runMode: false, + experimentalStrategy: "detect-flake-but-always-fail", + experimentalOptions: { + maxRetries: 2, + stopIfAnyPassed: true + } + }) + + Cypress.config('retries', { openMode: false, runMode: true, experimentalStrategy: "detect-flake-and-pass-on-threshold", experimentalOptions: { maxRetries: 2} }) // $ExpectError + Cypress.config('retries', { openMode: false, runMode: true, experimentalStrategy: "detect-flake-but-always-fail", experimentalOptions: { maxRetries: 2} }) // $ExpectError + + Cypress.config('retries', { openMode: false, runMode: true, experimentalStrategy: "detect-flake-and-pass-on-threshold", experimentalOptions: { passesRequired: 2} }) // $ExpectError + Cypress.config('retries', { openMode: false, runMode: true, experimentalStrategy: "detect-flake-but-always-fail", experimentalOptions: { stopIfAnyPassed: true} }) // $ExpectError +} + namespace CypressTraversalTests { cy.wrap({}).prevUntil('a') // $ExpectType Chainable> cy.wrap({}).prevUntil('#myItem') // $ExpectType Chainable> diff --git a/packages/app/src/settings/project/Experiments.vue b/packages/app/src/settings/project/Experiments.vue index 90b59334ddd2..b86489197e18 100644 --- a/packages/app/src/settings/project/Experiments.vue +++ b/packages/app/src/settings/project/Experiments.vue @@ -54,13 +54,33 @@ const props = defineProps<{ }>() const localExperiments = computed(() => { - return props.gql?.config ? (props.gql.config as CypressResolvedConfig).filter((item) => item.field.startsWith('experimental')).map((configItem) => { + // get experiments out of the config + const experimentalConfigurations = props.gql?.config ? (props.gql.config as CypressResolvedConfig).filter((item) => item.field.startsWith('experimental')) : [] + + // get experimental retry properties on the 'retries' config object. Mutate the experimentalConfigurations array as to not have side effects with props.gql.config + // TODO: remove this once experimentalRetries becomes GA. This is to be treated as a one off as supported nested experiments inside config is rare. + const { value: { experimentalStrategy, experimentalOptions, from } } = props.gql?.config.find((item) => item.field === 'retries') + + experimentalConfigurations.push({ + field: 'retries.experimentalStrategy', + from, + value: experimentalStrategy, + }) + + experimentalConfigurations.push({ + field: 'retries.experimentalOptions', + from, + value: experimentalOptions, + }) + // end TODO removal + + return experimentalConfigurations.map((configItem) => { return { key: configItem.field, name: t(`settingsPage.experiments.${configItem.field}.name`), enabled: !!configItem.value, description: t(`settingsPage.experiments.${configItem.field}.description`), } - }) : [] + }) }) diff --git a/packages/config/__snapshots__/validation.spec.ts.js b/packages/config/__snapshots__/validation.spec.ts.js index dbc4461334c4..80ec71d78a30 100644 --- a/packages/config/__snapshots__/validation.spec.ts.js +++ b/packages/config/__snapshots__/validation.spec.ts.js @@ -28,7 +28,7 @@ exports['browsers list with a string'] = { exports['invalid retry value'] = { 'key': 'mockConfigKey', 'value': '1', - 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls', + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', } exports['invalid retry object'] = { @@ -36,7 +36,7 @@ exports['invalid retry object'] = { 'value': { 'fakeMode': 1, }, - 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls', + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', } exports['not qualified url'] = { @@ -106,7 +106,7 @@ exports['null instead of a number'] = { exports['config/src/validation .isValidClientCertificatesSet returns error message for certs not passed as an array array 1'] = { 'key': 'mockConfigKey', 'value': '1', - 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls', + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', } exports['config/src/validation .isValidClientCertificatesSet returns error message for certs object without url 1'] = { @@ -332,3 +332,334 @@ exports['extraneous keys'] = { }, 'type': 'an object with keys `default` and `flaky`. Keys `default` and `flaky` must be integers greater than 0.', } + +exports['config/src/validation .isValidRetriesConfig experimental options fails with invalid strategy 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'experimentalStrategy': 'foo', + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with invalid strategy w/ other options 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'runMode': 1, + 'openMode': 0, + 'experimentalStrategy': 'bar', + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with maxRetries is negative 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'runMode': 1, + 'openMode': 0, + 'experimentalStrategy': 'detect-flake-and-pass-on-threshold', + 'experimentalOptions': { + 'maxRetries': -2, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with maxRetries is 0 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'runMode': 1, + 'openMode': 0, + 'experimentalStrategy': 'detect-flake-and-pass-on-threshold', + 'experimentalOptions': { + 'maxRetries': 0, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with maxRetries is floating 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'runMode': 1, + 'openMode': 0, + 'experimentalStrategy': 'detect-flake-and-pass-on-threshold', + 'experimentalOptions': { + 'maxRetries': 3.5, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: maxRetries is negative 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'experimentalStrategy': 'detect-flake-but-always-fail', + 'experimentalOptions': { + 'maxRetries': -2, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: maxRetries is 0 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'experimentalStrategy': 'detect-flake-but-always-fail', + 'experimentalOptions': { + 'maxRetries': 0, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail" maxRetries is floating 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'runMode': 1, + 'openMode': 0, + 'experimentalStrategy': 'detect-flake-but-always-fail', + 'experimentalOptions': { + 'maxRetries': 3.5, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold: maxRetries is negative 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'experimentalStrategy': 'detect-flake-and-pass-on-threshold', + 'experimentalOptions': { + 'maxRetries': -2, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold: maxRetries is 0 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'experimentalStrategy': 'detect-flake-and-pass-on-threshold', + 'experimentalOptions': { + 'maxRetries': 0, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold" maxRetries is floating 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'runMode': 1, + 'openMode': 0, + 'experimentalStrategy': 'detect-flake-and-pass-on-threshold', + 'experimentalOptions': { + 'maxRetries': 3.5, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: maxRetries is floating 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'experimentalStrategy': 'detect-flake-but-always-fail', + 'experimentalOptions': { + 'maxRetries': 3.5, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold: maxRetries is floating 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'experimentalStrategy': 'detect-flake-and-pass-on-threshold', + 'experimentalOptions': { + 'maxRetries': 3.5, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold passesRequired is negative 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'experimentalStrategy': 'detect-flake-and-pass-on-threshold', + 'experimentalOptions': { + 'maxRetries': 1, + 'passesRequired': -4, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold passesRequired is 0 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'experimentalStrategy': 'detect-flake-and-pass-on-threshold', + 'experimentalOptions': { + 'maxRetries': 1, + 'passesRequired': 0, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold passesRequired is floating 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'experimentalStrategy': 'detect-flake-and-pass-on-threshold', + 'experimentalOptions': { + 'maxRetries': 1, + 'passesRequired': 3.5, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold provides passesRequired without maxRetries 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'experimentalStrategy': 'detect-flake-and-pass-on-threshold', + 'experimentalOptions': { + 'passesRequired': 3, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold provides passesRequired that is greater than maxRetries 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'experimentalStrategy': 'detect-flake-and-pass-on-threshold', + 'experimentalOptions': { + 'maxRetries': 3, + 'passesRequired': 5, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold provides stopIfAnyPassed option 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'experimentalStrategy': 'detect-flake-and-pass-on-threshold', + 'experimentalOptions': { + 'maxRetries': 3, + 'stopIfAnyPassed': true, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail provides passesRequired option 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'experimentalStrategy': 'detect-flake-but-always-fail', + 'experimentalOptions': { + 'maxRetries': 3, + 'passesRequired': 2, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail provides stopIfAnyPassed without maxRetries 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'experimentalStrategy': 'detect-flake-but-always-fail', + 'experimentalOptions': { + 'stopIfAnyPassed': false, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail stopIfAnyPassed is a number (not coerced to a boolean 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'experimentalStrategy': 'detect-flake-but-always-fail', + 'experimentalOptions': { + 'maxRetries': 2, + 'stopIfAnyPassed': 1, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail stopIfAnyPassed is a number (0 and 1 work) 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'experimentalStrategy': 'detect-flake-but-always-fail', + 'experimentalOptions': { + 'maxRetries': 2, + 'stopIfAnyPassed': 2, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail stopIfAnyPassed is a number (0 and 1 do not work) 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'experimentalStrategy': 'detect-flake-but-always-fail', + 'experimentalOptions': { + 'maxRetries': 2, + 'stopIfAnyPassed': 1, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with invalid strategy w/ other options (valid) 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'runMode': true, + 'openMode': false, + 'experimentalStrategy': 'bar', + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: valid strategy w/ other invalid options with experiment 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'runMode': 1, + 'openMode': 0, + 'experimentalStrategy': 'detect-flake-but-always-fail', + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold: valid strategy w/ other invalid options with experiment 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'runMode': 1, + 'openMode': 0, + 'experimentalStrategy': 'detect-flake-and-pass-on-threshold', + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with experimentalStrategy is "detect-flake-but-always-fail" with only "maxRetries" in "experimentalOptions" 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'experimentalStrategy': 'detect-flake-but-always-fail', + 'experimentalOptions': { + 'maxRetries': 4, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} + +exports['config/src/validation .isValidRetriesConfig experimental options fails with experimentalStrategy is "detect-flake-and-pass-on-threshold" with only "maxRetries" in "experimentalOptions" 1'] = { + 'key': 'mockConfigKey', + 'value': { + 'experimentalStrategy': 'detect-flake-and-pass-on-threshold', + 'experimentalOptions': { + 'maxRetries': 4, + }, + }, + 'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy', +} diff --git a/packages/config/src/options.ts b/packages/config/src/options.ts index 2cef1c2a3688..6ad71b904abb 100644 --- a/packages/config/src/options.ts +++ b/packages/config/src/options.ts @@ -364,10 +364,32 @@ const driverConfigOptions: Array = [ validation: validate.isNumber, overrideLevel: 'any', }, { + /** + * if experimentalStrategy is `detect-flake-and-pass-on-threshold` + * an no experimentalOptions are configured, the following configuration + * should be implicitly used: + * experimentalStrategy: 'detect-flake-and-pass-on-threshold', + * experimentalOptions: { + * maxRetries: 2, + * passesRequired: 2 + * } + * + * if experimentalStrategy is `detect-flake-but-always-fail` + * an no experimentalOptions are configured, the following configuration + * should be implicitly used: + * experimentalStrategy: 'detect-flake-but-always-fail', + * experimentalOptions: { + * maxRetries: 2, + * stopIfAnyPassed: false + * } + */ name: 'retries', defaultValue: { runMode: 0, openMode: 0, + // these values MUST be populated in order to display the experiment correctly inside the project settings in open mode + experimentalStrategy: undefined, + experimentalOptions: undefined, }, validation: validate.isValidRetriesConfig, overrideLevel: 'any', diff --git a/packages/config/src/validation.ts b/packages/config/src/validation.ts index 180d59086105..0ef78a65b92d 100644 --- a/packages/config/src/validation.ts +++ b/packages/config/src/validation.ts @@ -120,8 +120,50 @@ export const isValidBrowserList = (_key: string, browsers: any): ErrResult | tru return true } +const isValidExperimentalRetryOptionsConfig = (options: any, strategy: 'detect-flake-but-always-fail' | 'detect-flake-and-pass-on-threshold'): boolean => { + if (options != null) { + // retries must be an integer of 1 or greater + const isValidMaxRetries = _.isInteger(options.maxRetries) && options.maxRetries > 0 + + if (!isValidMaxRetries) { + return false + } + + // if the strategy is 'detect-flake-but-always-fail', stopIfAnyPassed must be provided and must be a boolean + if (strategy === 'detect-flake-but-always-fail') { + if (options.passesRequired !== undefined) { + return false + } + + const isValidStopIfAnyPasses = _.isBoolean(options.stopIfAnyPassed) + + if (!isValidStopIfAnyPasses) { + return false + } + } + + // if the strategy is 'detect-flake-and-pass-on-threshold', passesRequired must be provided and must be an integer greater than 0 + if (strategy === 'detect-flake-and-pass-on-threshold') { + if (options.stopIfAnyPassed !== undefined) { + return false + } + + const isValidPassesRequired = _.isInteger(options.passesRequired) && options.passesRequired > 0 && options.passesRequired <= options.maxRetries + + if (!isValidPassesRequired) { + return false + } + } + } + + return true +} + export const isValidRetriesConfig = (key: string, value: any): ErrResult | true => { const optionalKeys = ['runMode', 'openMode'] + const experimentalOptions = ['experimentalStrategy', 'experimentalOptions'] + const experimentalStrategyOptions = ['detect-flake-but-always-fail', 'detect-flake-and-pass-on-threshold'] + const isValidRetryValue = (val: any) => _.isNull(val) || (Number.isInteger(val) && val >= 0) const optionalKeysAreValid = (val: any, k: string) => optionalKeys.includes(k) && isValidRetryValue(val) @@ -129,11 +171,37 @@ export const isValidRetriesConfig = (key: string, value: any): ErrResult | true return true } - if (_.isObject(value) && _.every(value, optionalKeysAreValid)) { - return true + if (_.isObject(value)) { + const traditionalConfigOptions = _.omit(value, experimentalOptions) + const experimentalConfigOptions = _.pick(value, experimentalOptions) + + const isTraditionalConfigValid = _.every(traditionalConfigOptions, optionalKeysAreValid) + + // if optionalKeys are only present and are valid, return true. + // The defaults for 'experimentalStrategy' and 'experimentalOptions' are undefined, but the keys exist, so we need to check for this + if (isTraditionalConfigValid && !Object.keys(experimentalConfigOptions).filter((key) => experimentalConfigOptions[key] !== undefined).length) { + return true + } + + // check experimental configuration. experimentalStrategy MUST be present if experimental config is provided and set to one of the provided enumerations + if (experimentalConfigOptions.experimentalStrategy) { + // make sure the strategy provided is one of our valid enums + const isValidStrategy = experimentalStrategyOptions.includes(experimentalConfigOptions.experimentalStrategy) + + // if a strategy is provided, and traditional options are also provided, such as runMode and openMode, then these values need to be booleans + const openAndRunModeConfigOptions = _.pick(value, optionalKeys) + const isValidRunAndOpenModeConfigWithStrategy = _.every(openAndRunModeConfigOptions, _.isBoolean) + + // if options aren't present (either undefined or null) or are configured correctly, return true + if (isValidStrategy && isValidRunAndOpenModeConfigWithStrategy && ( + experimentalConfigOptions.experimentalOptions == null || + isValidExperimentalRetryOptionsConfig(experimentalConfigOptions.experimentalOptions, experimentalConfigOptions.experimentalStrategy))) { + return true + } + } } - return errMsg(key, value, 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls') + return errMsg(key, value, 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy') } /** diff --git a/packages/config/test/project/utils.spec.ts b/packages/config/test/project/utils.spec.ts index 7342c6557527..502795dafe6f 100644 --- a/packages/config/test/project/utils.spec.ts +++ b/packages/config/test/project/utils.spec.ts @@ -1113,7 +1113,7 @@ describe('config/src/project/utils', () => { reporterOptions: { value: null, from: 'default' }, requestTimeout: { value: 5000, from: 'default' }, responseTimeout: { value: 30000, from: 'default' }, - retries: { value: { runMode: 0, openMode: 0 }, from: 'default' }, + retries: { value: { runMode: 0, openMode: 0, experimentalStrategy: undefined, experimentalOptions: undefined }, from: 'default' }, screenshotOnRunFailure: { value: true, from: 'default' }, screenshotsFolder: { value: 'cypress/screenshots', from: 'default' }, specPattern: { value: 'cypress/e2e/**/*.cy.{js,jsx,ts,tsx}', from: 'default' }, @@ -1234,7 +1234,7 @@ describe('config/src/project/utils', () => { reporterOptions: { value: null, from: 'default' }, requestTimeout: { value: 5000, from: 'default' }, responseTimeout: { value: 30000, from: 'default' }, - retries: { value: { runMode: 0, openMode: 0 }, from: 'default' }, + retries: { value: { runMode: 0, openMode: 0, experimentalStrategy: undefined, experimentalOptions: undefined }, from: 'default' }, screenshotOnRunFailure: { value: true, from: 'default' }, screenshotsFolder: { value: 'cypress/screenshots', from: 'default' }, slowTestThreshold: { value: 10000, from: 'default' }, diff --git a/packages/config/test/validation.spec.ts b/packages/config/test/validation.spec.ts index df538d799da8..de5d9c450c96 100644 --- a/packages/config/test/validation.spec.ts +++ b/packages/config/test/validation.spec.ts @@ -166,6 +166,289 @@ describe('config/src/validation', () => { expect(result).to.not.be.true snapshot('invalid retry object', result) }) + + it('returns true for valid retry object with experimental keys (default)', () => { + let result = validation.isValidRetriesConfig(mockKey, { + openMode: 0, + runMode: 0, + experimentalStrategy: undefined, + experimentalOptions: undefined, + }) + + expect(result).to.be.true + }) + + describe('experimental options', () => { + describe('passes with', () => { + ['detect-flake-but-always-fail', 'detect-flake-and-pass-on-threshold'].forEach((strategy) => { + it(`experimentalStrategy is "${strategy}" with no "experimentalOptions" & valid runMode and openMode`, () => { + let result = validation.isValidRetriesConfig(mockKey, { + runMode: true, + openMode: false, + experimentalStrategy: strategy, + }) + + expect(result).to.be.true + + result = validation.isValidRetriesConfig(mockKey, { + experimentalStrategy: strategy, + }) + + expect(result).to.be.true + }) + }) + + it('experimentalStrategy is "detect-flake-but-always-fail" and has option "stopIfAnyPassed"', () => { + let result = validation.isValidRetriesConfig(mockKey, { + runMode: true, + openMode: false, + experimentalStrategy: 'detect-flake-but-always-fail', + experimentalOptions: { + maxRetries: 1, + stopIfAnyPassed: true, + }, + }) + + expect(result).to.be.true + + result = validation.isValidRetriesConfig(mockKey, { + experimentalStrategy: 'detect-flake-but-always-fail', + experimentalOptions: { + maxRetries: 4, + stopIfAnyPassed: false, + }, + }) + + expect(result).to.be.true + }) + + it('experimentalStrategy is "detect-flake-and-pass-on-threshold" and has option "passesRequired"', () => { + let result = validation.isValidRetriesConfig(mockKey, { + runMode: true, + openMode: false, + experimentalStrategy: 'detect-flake-and-pass-on-threshold', + experimentalOptions: { + maxRetries: 1, + passesRequired: 1, + }, + }) + + expect(result).to.be.true + + result = validation.isValidRetriesConfig(mockKey, { + experimentalStrategy: 'detect-flake-and-pass-on-threshold', + experimentalOptions: { + maxRetries: 4, + passesRequired: 2, + }, + }) + + expect(result).to.be.true + }) + }) + + describe('fails with', () => { + it('invalid strategy', () => { + const result = validation.isValidRetriesConfig(mockKey, { + experimentalStrategy: 'foo', + }) + + expect(result).to.not.be.true + snapshot(result) + }) + + it('invalid strategy w/ other options (valid)', () => { + const result = validation.isValidRetriesConfig(mockKey, { + runMode: true, + openMode: false, + experimentalStrategy: 'bar', + }) + + expect(result).to.not.be.true + snapshot(result) + }) + + ;['detect-flake-but-always-fail', 'detect-flake-and-pass-on-threshold'].forEach((strategy) => { + it(`${strategy}: valid strategy w/ other invalid options with experiment`, () => { + const result = validation.isValidRetriesConfig(mockKey, { + runMode: 1, + openMode: 0, + experimentalStrategy: strategy, + }) + + expect(result).to.not.be.true + snapshot(result) + }) + + it(`${strategy}: maxRetries is negative`, () => { + const result = validation.isValidRetriesConfig(mockKey, { + experimentalStrategy: strategy, + experimentalOptions: { + maxRetries: -2, + }, + }) + + expect(result).to.not.be.true + snapshot(result) + }) + + it(`${strategy}: maxRetries is 0`, () => { + const result = validation.isValidRetriesConfig(mockKey, { + experimentalStrategy: strategy, + experimentalOptions: { + maxRetries: 0, + }, + }) + + expect(result).to.not.be.true + snapshot(result) + }) + + it(`${strategy}: maxRetries is floating`, () => { + const result = validation.isValidRetriesConfig(mockKey, { + experimentalStrategy: strategy, + experimentalOptions: { + maxRetries: 3.5, + }, + }) + + expect(result).to.not.be.true + snapshot(result) + }) + + it(`experimentalStrategy is "${strategy}" with only "maxRetries" in "experimentalOptions"`, () => { + const result = validation.isValidRetriesConfig(mockKey, { + experimentalStrategy: strategy, + experimentalOptions: { + maxRetries: 4, + }, + }) + + expect(result).to.not.be.true + snapshot(result) + }) + }) + + describe('detect-flake-and-pass-on-threshold', () => { + it(`passesRequired is negative`, () => { + const result = validation.isValidRetriesConfig(mockKey, { + experimentalStrategy: 'detect-flake-and-pass-on-threshold', + experimentalOptions: { + maxRetries: 1, + passesRequired: -4, + }, + }) + + expect(result).to.not.be.true + snapshot(result) + }) + + it(`passesRequired is 0`, () => { + const result = validation.isValidRetriesConfig(mockKey, { + experimentalStrategy: 'detect-flake-and-pass-on-threshold', + experimentalOptions: { + maxRetries: 1, + passesRequired: 0, + }, + }) + + expect(result).to.not.be.true + snapshot(result) + }) + + it(`passesRequired is floating`, () => { + const result = validation.isValidRetriesConfig(mockKey, { + experimentalStrategy: 'detect-flake-and-pass-on-threshold', + experimentalOptions: { + maxRetries: 1, + passesRequired: 3.5, + }, + }) + + expect(result).to.not.be.true + snapshot(result) + }) + + it('provides passesRequired without maxRetries', () => { + const result = validation.isValidRetriesConfig(mockKey, { + experimentalStrategy: 'detect-flake-and-pass-on-threshold', + experimentalOptions: { + passesRequired: 3, + }, + }) + + expect(result).to.not.be.true + snapshot(result) + }) + + it('provides passesRequired that is greater than maxRetries', () => { + const result = validation.isValidRetriesConfig(mockKey, { + experimentalStrategy: 'detect-flake-and-pass-on-threshold', + experimentalOptions: { + maxRetries: 3, + passesRequired: 5, + }, + }) + + expect(result).to.not.be.true + snapshot(result) + }) + + it('provides stopIfAnyPassed option', () => { + const result = validation.isValidRetriesConfig(mockKey, { + experimentalStrategy: 'detect-flake-and-pass-on-threshold', + experimentalOptions: { + maxRetries: 3, + stopIfAnyPassed: true, + }, + }) + + expect(result).to.not.be.true + snapshot(result) + }) + }) + + describe('detect-flake-but-always-fail', () => { + it('provides passesRequired option', () => { + const result = validation.isValidRetriesConfig(mockKey, { + experimentalStrategy: 'detect-flake-but-always-fail', + experimentalOptions: { + maxRetries: 3, + passesRequired: 2, + }, + }) + + expect(result).to.not.be.true + snapshot(result) + }) + + it('provides stopIfAnyPassed without maxRetries', () => { + const result = validation.isValidRetriesConfig(mockKey, { + experimentalStrategy: 'detect-flake-but-always-fail', + experimentalOptions: { + stopIfAnyPassed: false, + }, + }) + + expect(result).to.not.be.true + snapshot(result) + }) + + it('stopIfAnyPassed is a number (0 and 1 do not work)', () => { + const result = validation.isValidRetriesConfig(mockKey, { + experimentalStrategy: 'detect-flake-but-always-fail', + experimentalOptions: { + maxRetries: 2, + stopIfAnyPassed: 1, + }, + }) + + expect(result).to.not.be.true + snapshot(result) + }) + }) + }) + }) }) describe('.isPlainObject', () => { diff --git a/packages/frontend-shared/src/locales/en-US.json b/packages/frontend-shared/src/locales/en-US.json index da26aa6f1dc9..5432660213dd 100644 --- a/packages/frontend-shared/src/locales/en-US.json +++ b/packages/frontend-shared/src/locales/en-US.json @@ -590,6 +590,16 @@ "name": "Modify obstructive third party code", "description": "Applies `modifyObstructiveCode` to third party `.html` and `.js`, removes subresource integrity, and modifies the user agent in Electron." }, + "retries": { + "experimentalStrategy": { + "name": "Retries Strategy", + "description": "Applies a strategy for test retries according to your \"flake tolerance\"; options are `detect-flake-but-always-fail` or `detect-flake-and-pass-on-threshold`." + }, + "experimentalOptions": { + "name": "Retries Strategy Options", + "description": "Sets retries strategy-specific options like `maxRetries`, `passesRequired`, and `stopIfAnyPassed`." + } + }, "experimentalSingleTabRunMode": { "name": "Single tab run mode", "description": "Runs all component specs in a single tab, trading spec isolation for faster run mode execution." diff --git a/packages/server/test/unit/config_spec.js b/packages/server/test/unit/config_spec.js index 5c3fae8fc6f9..1836e3ca19f7 100644 --- a/packages/server/test/unit/config_spec.js +++ b/packages/server/test/unit/config_spec.js @@ -897,7 +897,7 @@ describe('lib/config', () => { }) context('retries', () => { - const retriesError = 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls' + const retriesError = 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy' // need to keep the const here or it'll get stripped by the build // eslint-disable-next-line no-unused-vars diff --git a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js index 9a1f6a5374c9..d8331f4f95e4 100644 --- a/system-tests/__snapshots__/testConfigOverrides_spec.ts.js +++ b/system-tests/__snapshots__/testConfigOverrides_spec.ts.js @@ -320,7 +320,7 @@ https://on.cypress.io/config runs: CypressError: The config passed to your suite-level overrides has the following validation error: -Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. +Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy. Instead the value was: \`"1"\` @@ -369,7 +369,7 @@ https://on.cypress.io/config "before all" hook for "test config override throws error": CypressError: The config passed to your test-level overrides has the following validation error: -Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. +Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy. Instead the value was: \`"1"\` @@ -383,7 +383,7 @@ Because this error occurred during a \`before all\` hook we are skipping the rem test config override throws error: CypressError: The config passed to your test-level overrides has the following validation error: -Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. +Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy. Instead the value was: \`"1"\` @@ -528,7 +528,7 @@ Instead the value was: \`"null"\` runs: CypressError: The config passed to your suite-level overrides has the following validation error: -Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. +Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy. Instead the value was: \`"1"\` @@ -581,7 +581,7 @@ https://on.cypress.io/config "before all" hook for "test config override throws error": CypressError: The config passed to your test-level overrides has the following validation error: -Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. +Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy. Instead the value was: \`"1"\` @@ -596,7 +596,7 @@ Because this error occurred during a \`before all\` hook we are skipping the rem test config override throws error: CypressError: The config passed to your test-level overrides has the following validation error: -Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. +Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy. Instead the value was: \`"1"\` @@ -676,7 +676,7 @@ exports['testConfigOverrides / correctly fails when invalid config values for it throws error at the correct line number: CypressError: The config passed to your suite-level overrides has the following validation error: -Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. +Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy. Instead the value was: \`"1"\` @@ -992,7 +992,7 @@ https://on.cypress.io/config runs: CypressError: The config passed to your suite-level overrides has the following validation error: -Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. +Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy. Instead the value was: \`"1"\` @@ -1037,7 +1037,7 @@ https://on.cypress.io/config "before all" hook for "test config override throws error": CypressError: The config passed to your test-level overrides has the following validation error: -Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. +Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy. Instead the value was: \`"1"\` @@ -1050,7 +1050,7 @@ Because this error occurred during a \`before all\` hook we are skipping the rem test config override throws error: CypressError: The config passed to your test-level overrides has the following validation error: -Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. +Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy. Instead the value was: \`"1"\` @@ -1194,7 +1194,7 @@ Instead the value was: \`"null"\` runs: CypressError: The config passed to your suite-level overrides has the following validation error: -Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. +Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy. Instead the value was: \`"1"\` @@ -1243,7 +1243,7 @@ https://on.cypress.io/config "before all" hook for "test config override throws error": CypressError: The config passed to your test-level overrides has the following validation error: -Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. +Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy. Instead the value was: \`"1"\` @@ -1257,7 +1257,7 @@ Because this error occurred during a \`before all\` hook we are skipping the rem test config override throws error: CypressError: The config passed to your test-level overrides has the following validation error: -Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. +Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy. Instead the value was: \`"1"\` @@ -1336,7 +1336,7 @@ exports['testConfigOverrides / correctly fails when invalid config values for it throws error at the correct line number: CypressError: The config passed to your suite-level overrides has the following validation error: -Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers or nulls. +Expected \`retries\` to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy. Instead the value was: \`"1"\`