chore(webkit): add video recording (#23579)

circle.yml

@@ -178,6 +178,14 @@ commands:
                   mv ~/cypress/system-tests/node_modules /tmp/node_modules_cache/system-tests_node_modules
                   mv ~/cypress/globbed_node_modules /tmp/node_modules_cache/globbed_node_modules
 
+  install-webkit-deps:
+    steps:
+      - run:
+          name: Install WebKit dependencies
+          command: |
+            npx playwright install webkit
+            npx playwright install-deps webkit
+
   build-and-persist:
     description: Save entire folder as artifact for other jobs to run without reinstalling
     steps:
@@ -462,6 +470,11 @@ commands:
             - install-chrome:
                 channel: <<parameters.install-chrome-channel>>
                 version: $(node ./scripts/get-browser-version.js chrome:<<parameters.install-chrome-channel>>)
+      - when:
+          condition:
+            equal: [ webkit, << parameters.browser >> ]
+          steps:
+            - install-webkit-deps
       - run:
           name: Run driver tests in Cypress
           environment:
@@ -470,11 +483,6 @@ commands:
             echo Current working directory is $PWD
             echo Total containers $CIRCLE_NODE_TOTAL
 
-            if [[ "<<parameters.browser>>" = "webkit" ]]; then
-              npx playwright install webkit
-              npx playwright install-deps webkit
-            fi
-
             if [[ -v MAIN_RECORD_KEY ]]; then
               # internal PR
               if <<parameters.experimentalSessionAndOrigin>>; then
@@ -610,6 +618,11 @@ commands:
     steps:
       - restore_cached_workspace
       - restore_cached_system_tests_deps
+      - when:
+          condition:
+            equal: [ webkit, << parameters.browser >> ]
+          steps:
+            - install-webkit-deps
       - run:
           name: Run system tests
           command: |
@@ -1448,6 +1461,13 @@ jobs:
       - run-system-tests:
           browser: firefox
 
+  system-tests-webkit:
+    <<: *defaults
+    parallelism: 8
+    steps:
+      - run-system-tests:
+          browser: webkit
+
   system-tests-non-root:
     <<: *defaults
     steps:
@@ -2363,6 +2383,10 @@ linux-x64-workflow: &linux-x64-workflow
         context: test-runner:performance-tracking
         requires:
           - system-tests-node-modules-install
+    - system-tests-webkit:
+        context: test-runner:performance-tracking
+        requires:
+          - system-tests-node-modules-install
     - system-tests-non-root:
         context: test-runner:performance-tracking
         executor: non-root-docker-user

packages/server/lib/browsers/chrome.ts

@@ -15,12 +15,11 @@ import { fs } from '../util/fs'
 import { CdpAutomation, screencastOpts } from './cdp_automation'
 import * as protocol from './protocol'
 import utils from './utils'
-import type { Browser } from './types'
+import type { Browser, BrowserInstance } from './types'
 import { BrowserCriClient } from './browser-cri-client'
-import type { LaunchedBrowser } from '@packages/launcher/lib/browsers'
 import type { CriClient } from './cri-client'
 import type { Automation } from '../automation'
-import type { BrowserLaunchOpts, BrowserNewTabOpts, WriteVideoFrame } from '@packages/types'
+import type { BrowserLaunchOpts, BrowserNewTabOpts, RunModeVideoApi } from '@packages/types'
 
 const debug = debugModule('cypress:server:browsers:chrome')
 
@@ -249,10 +248,12 @@ const _disableRestorePagesPrompt = function (userDir) {
   .catch(() => { })
 }
 
-async function _recordVideo (cdpAutomation: CdpAutomation, writeVideoFrame: WriteVideoFrame, browserMajorVersion: number) {
-  const opts = browserMajorVersion >= CHROME_VERSION_WITH_FPS_INCREASE ? screencastOpts() : screencastOpts(1)
+async function _recordVideo (cdpAutomation: CdpAutomation, videoOptions: RunModeVideoApi, browserMajorVersion: number) {
+  const screencastOptions = browserMajorVersion >= CHROME_VERSION_WITH_FPS_INCREASE ? screencastOpts() : screencastOpts(1)
 
-  await cdpAutomation.startVideoRecording(writeVideoFrame, opts)
+  const { writeVideoFrame } = await videoOptions.useFfmpegVideoController()
+
+  await cdpAutomation.startVideoRecording(writeVideoFrame, screencastOptions)
 }
 
 // a utility function that navigates to the given URL
@@ -552,7 +553,7 @@ export = {
 
     if (!options.url) throw new Error('Missing url in connectToNewSpec')
 
-    await this.attachListeners(browser, options.url, pageCriClient, automation, options)
+    await this.attachListeners(options.url, pageCriClient, automation, options)
   },
 
   async connectToExisting (browser: Browser, options: BrowserLaunchOpts, automation: Automation) {
@@ -570,17 +571,21 @@ export = {
     await this._setAutomation(pageCriClient, automation, browserCriClient.resetBrowserTargets, options)
   },
 
-  async attachListeners (browser: Browser, url: string, pageCriClient: CriClient, automation: Automation, options: BrowserLaunchOpts | BrowserNewTabOpts) {
+  async attachListeners (url: string, pageCriClient: CriClient, automation: Automation, options: BrowserLaunchOpts | BrowserNewTabOpts) {
+    const browserCriClient = this._getBrowserCriClient()
+
     if (!browserCriClient) throw new Error('Missing browserCriClient in attachListeners')
 
+    debug('attaching listeners to chrome %o', { url, options })
+
     const cdpAutomation = await this._setAutomation(pageCriClient, automation, browserCriClient.resetBrowserTargets, options)
 
     await pageCriClient.send('Page.enable')
 
     await options['onInitializeNewBrowserTab']?.()
 
     await Promise.all([
-      options.writeVideoFrame && this._recordVideo(cdpAutomation, options.writeVideoFrame, browser.majorVersion),
+      options.videoApi && this._recordVideo(cdpAutomation, options.videoApi, Number(options.browser.majorVersion)),
       this._handleDownloads(pageCriClient, options.downloadsFolder, automation),
     ])
 
@@ -590,9 +595,11 @@ export = {
       await this._handlePausedRequests(pageCriClient)
       _listenForFrameTreeChanges(pageCriClient)
     }
+
+    return cdpAutomation
   },
 
-  async open (browser: Browser, url, options: BrowserLaunchOpts, automation: Automation): Promise<LaunchedBrowser> {
+  async open (browser: Browser, url, options: BrowserLaunchOpts, automation: Automation): Promise<BrowserInstance> {
     const { isTextTerminal } = options
 
     const userDir = utils.getProfileDir(browser, isTextTerminal)
@@ -644,7 +651,7 @@ export = {
     // first allows us to connect the remote interface,
     // start video recording and then
     // we will load the actual page
-    const launchedBrowser = await launch(browser, 'about:blank', port, args) as LaunchedBrowser & { browserCriClient: BrowserCriClient}
+    const launchedBrowser = await launch(browser, 'about:blank', port, args) as unknown as BrowserInstance & { browserCriClient: BrowserCriClient}
 
     la(launchedBrowser, 'did not get launched browser instance')
 
@@ -671,7 +678,6 @@ export = {
 
     launchedBrowser.browserCriClient = browserCriClient
 
-    /* @ts-expect-error */
     launchedBrowser.kill = (...args) => {
       debug('closing remote interface client')
 
@@ -686,7 +692,7 @@ export = {
 
     const pageCriClient = await browserCriClient.attachToTargetUrl('about:blank')
 
-    await this.attachListeners(browser, url, pageCriClient, automation, options)
+    await this.attachListeners(url, pageCriClient, automation, options)
 
     // return the launched browser process
     // with additional method to close the remote connection

packages/server/lib/browsers/electron.ts

@@ -11,7 +11,7 @@ import * as errors from '../errors'
 import type { Browser, BrowserInstance } from './types'
 import type { BrowserWindow, WebContents } from 'electron'
 import type { Automation } from '../automation'
-import type { BrowserLaunchOpts, Preferences } from '@packages/types'
+import type { BrowserLaunchOpts, Preferences, RunModeVideoApi } from '@packages/types'
 
 // TODO: unmix these two types
 type ElectronOpts = Windows.WindowOptions & BrowserLaunchOpts
@@ -44,7 +44,7 @@ const tryToCall = function (win, method) {
   }
 }
 
-const _getAutomation = async function (win, options, parent) {
+const _getAutomation = async function (win, options: BrowserLaunchOpts, parent) {
   async function sendCommand (method: CdpCommand, data?: object) {
     return tryToCall(win, () => {
       return win.webContents.debugger.sendCommand
@@ -72,7 +72,7 @@ const _getAutomation = async function (win, options, parent) {
         // after upgrading to Electron 8, CDP screenshots can hang if a screencast is not also running
         // workaround: start and stop screencasts between screenshots
         // @see https://github.com/cypress-io/cypress/pull/6555#issuecomment-596747134
-        if (!options.writeVideoFrame) {
+        if (!options.videoApi) {
           await sendCommand('Page.startScreencast', screencastOpts())
           const ret = await fn(message, data)
 
@@ -109,6 +109,12 @@ function _installExtensions (win: BrowserWindow, extensionPaths: string[], optio
   }))
 }
 
+async function recordVideo (cdpAutomation: CdpAutomation, videoApi: RunModeVideoApi) {
+  const { writeVideoFrame } = await videoApi.useFfmpegVideoController()
+
+  await cdpAutomation.startVideoRecording(writeVideoFrame)
+}
+
 export = {
   _defaultOptions (projectRoot: string | undefined, state: Preferences, options: BrowserLaunchOpts, automation: Automation): ElectronOpts {
     const _this = this
@@ -179,7 +185,7 @@ export = {
 
   _getAutomation,
 
-  async _render (url: string, automation: Automation, preferences, options: { projectRoot?: string, isTextTerminal: boolean }) {
+  async _render (url: string, automation: Automation, preferences, options: ElectronOpts) {
     const win = Windows.create(options.projectRoot, preferences)
 
     if (preferences.browser.isHeadless) {
@@ -192,7 +198,7 @@ export = {
       win.maximize()
     }
 
-    const launched = await this._launch(win, url, automation, preferences)
+    const launched = await this._launch(win, url, automation, preferences, options.videoApi)
 
     automation.use(await _getAutomation(win, preferences, automation))
 
@@ -225,7 +231,7 @@ export = {
     return this._launch(win, url, automation, electronOptions)
   },
 
-  async _launch (win: BrowserWindow, url: string, automation: Automation, options) {
+  async _launch (win: BrowserWindow, url: string, automation: Automation, options: ElectronOpts, videoApi?: RunModeVideoApi) {
     if (options.show) {
       menu.set({ withInternalDevTools: true })
     }
@@ -239,9 +245,7 @@ export = {
 
     this._attachDebugger(win.webContents)
 
-    let ua
-
-    ua = options.userAgent
+    const ua = options.userAgent
 
     if (ua) {
       this._setUserAgent(win.webContents, ua)
@@ -273,8 +277,9 @@ export = {
     const cdpAutomation = await this._getAutomation(win, options, automation)
 
     automation.use(cdpAutomation)
+
     await Promise.all([
-      options.writeVideoFrame && cdpAutomation.startVideoRecording(options.writeVideoFrame),
+      videoApi && recordVideo(cdpAutomation, videoApi),
       this._handleDownloads(win, options.downloadsFolder, automation),
     ])
 
@@ -445,7 +450,7 @@ export = {
     })
   },
 
-  async connectToNewSpec (browser, options, automation) {
+  async connectToNewSpec (browser: Browser, options: ElectronOpts, automation: Automation) {
     if (!options.url) throw new Error('Missing url in connectToNewSpec')
 
     await this.open(browser, options.url, options, automation)
@@ -480,10 +485,7 @@ export = {
 
     debug('launching browser window to url: %s', url)
 
-    const win = await this._render(url, automation, preferences, {
-      projectRoot: options.projectRoot,
-      isTextTerminal: options.isTextTerminal,
-    })
+    const win = await this._render(url, automation, preferences, electronOptions)
 
     await _installExtensions(win, launchOptions.extensions, electronOptions)
 

packages/server/lib/browsers/firefox.ts

@@ -4,7 +4,7 @@ import Debug from 'debug'
 import getPort from 'get-port'
 import path from 'path'
 import urlUtil from 'url'
-import { debug as launcherDebug, launch, LaunchedBrowser } from '@packages/launcher/lib/browsers'
+import { debug as launcherDebug, launch } from '@packages/launcher/lib/browsers'
 import { doubleEscape } from '@packages/launcher/lib/windows'
 import FirefoxProfile from 'firefox-profile'
 import * as errors from '../errors'
@@ -20,7 +20,7 @@ import type { BrowserCriClient } from './browser-cri-client'
 import type { Automation } from '../automation'
 import { getCtx } from '@packages/data-context'
 import { getError } from '@packages/errors'
-import type { BrowserLaunchOpts, BrowserNewTabOpts } from '@packages/types'
+import type { BrowserLaunchOpts, BrowserNewTabOpts, RunModeVideoApi } from '@packages/types'
 
 const debug = Debug('cypress:server:browsers:firefox')
 
@@ -379,6 +379,12 @@ export function connectToExisting () {
   getCtx().onWarning(getError('UNEXPECTED_INTERNAL_ERROR', new Error('Attempting to connect to existing browser for Cypress in Cypress which is not yet implemented for firefox')))
 }
 
+async function recordVideo (videoApi: RunModeVideoApi) {
+  const { writeVideoFrame } = await videoApi.useFfmpegVideoController({ webmInput: true })
+
+  videoApi.onProjectCaptureVideoFrames(writeVideoFrame)
+}
+
 export async function open (browser: Browser, url: string, options: BrowserLaunchOpts, automation: Automation): Promise<BrowserInstance> {
   // see revision comment here https://wiki.mozilla.org/index.php?title=WebDriver/RemoteProtocol&oldid=1234946
   const hasCdp = browser.majorVersion >= 86
@@ -456,6 +462,7 @@ export async function open (browser: Browser, url: string, options: BrowserLaunc
     utils.ensureCleanCache(browser, options.isTextTerminal),
     utils.writeExtension(browser, options.isTextTerminal, options.proxyUrl, options.socketIoRoute),
     utils.executeBeforeBrowserLaunch(browser, defaultLaunchOptions, options),
+    options.videoApi && recordVideo(options.videoApi),
   ])
 
   if (Array.isArray(launchOptions.extensions)) {
@@ -529,7 +536,7 @@ export async function open (browser: Browser, url: string, options: BrowserLaunc
     // user can overwrite this default with these env vars or --height, --width arguments
     MOZ_HEADLESS_WIDTH: '1280',
     MOZ_HEADLESS_HEIGHT: '721',
-  }) as LaunchedBrowser & { browserCriClient: BrowserCriClient}
+  }) as unknown as BrowserInstance
 
   try {
     browserCriClient = await firefoxUtil.setup({ automation, extensions: launchOptions.extensions, url, foxdriverPort, marionettePort, remotePort, onError: options.onError, options })
@@ -543,7 +550,6 @@ export async function open (browser: Browser, url: string, options: BrowserLaunc
     // monkey-patch the .kill method to that the CDP connection is closed
     const originalBrowserKill = browserInstance.kill
 
-    /* @ts-expect-error */
     browserInstance.kill = (...args) => {
       debug('closing remote interface client')