From 36a138a421117a087aa73929371d45fc6b1fb4da Mon Sep 17 00:00:00 2001 From: Sam Ko Date: Fri, 10 Jan 2025 11:34:09 -0800 Subject: [PATCH 1/9] chore(github): update how we handle stale issues (#73488) ## Why? We currently have no automated process for handling stale issues. This PR adds an automated process for this. --- .github/actions/next-repo-actions/package.json | 2 +- .github/workflows/issue_stale.yml | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/actions/next-repo-actions/package.json b/.github/actions/next-repo-actions/package.json index 42c76a0d1bf04..fcfc1bf2d06c5 100644 --- a/.github/actions/next-repo-actions/package.json +++ b/.github/actions/next-repo-actions/package.json @@ -1,6 +1,6 @@ { "private": true, - "description": "Notify Next.js team about pending PRs and popular issues", + "description": "A variety of functions to help with triaging issues, PRs, and feature requests in the Next.js repo.", "scripts": { "build-issues-by-version": "ncc build src/issues-by-version.ts -m -o dist/issues-by-version --license licenses.txt", "build-issues": "ncc build src/popular-issues.mjs -m -o dist/issues --license licenses.txt", diff --git a/.github/workflows/issue_stale.yml b/.github/workflows/issue_stale.yml index 4cc667475eb39..e4e67f56a584b 100644 --- a/.github/workflows/issue_stale.yml +++ b/.github/workflows/issue_stale.yml @@ -10,6 +10,19 @@ jobs: runs-on: ubuntu-latest if: github.repository_owner == 'vercel' steps: + - uses: actions/stale@v9 + id: issue-stale + name: 'Mark stale issues, close stale issues' + with: + repo-token: ${{ secrets.STALE_TOKEN }} + ascending: true + days-before-issue-stale: 730 # issues with no activity in over two years + days-before-issue-close: 7 + remove-issue-stale-when-updated: true + stale-issue-label: 'stale' + stale-issue-message: 'This issue has been automatically marked as stale due to two years of inactivity. It will be closed in 7 days unless there’s further input. If you believe this issue is still relevant, please leave a comment or provide updated details. Thank you.' + close-issue-message: 'This issue has been automatically closed due to two years of inactivity. If you’re still experiencing a similar problem or have additional details to share, please open a new issue following our current issue template. Your updated report helps us investigate and address concerns more efficiently. Thank you for your understanding!' + operations-per-run: 300 # 1 operation per 100 issues, the rest is to label/comment/close - uses: actions/stale@v9 id: stale-no-repro name: 'Close stale issues with no reproduction' @@ -21,7 +34,6 @@ jobs: days-before-issue-stale: 2 days-before-pr-close: -1 days-before-pr-stale: -1 - exempt-issue-labels: 'blocked,must,should,keep' operations-per-run: 300 # 1 operation per 100 issues, the rest is to label/comment/close - uses: actions/stale@v9 id: stale-simple-repro @@ -34,7 +46,6 @@ jobs: days-before-issue-stale: 14 days-before-pr-close: -1 days-before-pr-stale: -1 - exempt-issue-labels: 'blocked,must,should,keep' operations-per-run: 300 # 1 operation per 100 issues, the rest is to label/comment/close - uses: actions/stale@v9 id: stale-no-canary @@ -47,5 +58,4 @@ jobs: days-before-issue-stale: 14 days-before-pr-close: -1 days-before-pr-stale: -1 - exempt-issue-labels: 'blocked,must,should,keep' operations-per-run: 300 # 1 operation per 100 issues, the rest is to label/comment/close From d483dfac1e9407b1c40bf7d5e589a45c26bf192f Mon Sep 17 00:00:00 2001 From: Jiachi Liu Date: Fri, 10 Jan 2025 20:43:48 +0100 Subject: [PATCH 2/9] [metadata] add option of configuring ua of async metadata (#74594) ### What Introduce an experimental config `experimental.htmlLimitedBots` (type `Regex`) which can used for controlling the metadata streaming behavior for different bots, if the bots user agent is matching the regex, it will serve blocking metadata forcedly. This gives users some control to opt-out streaming metadata for some clients or bots which cannot handle it. Output another manifest `response-config-manfiest.json` containing the customized UA, which is potentially used for the platform serving Next.js to control the returned response for certain UA. But it's mostly for PPR case which we'll tackle later. Closes NDX-635 Closes NDX-599 --- packages/next/src/build/index.ts | 23 ++++++++++ packages/next/src/build/utils.ts | 4 +- packages/next/src/export/index.ts | 1 + packages/next/src/server/app-render/types.ts | 1 + packages/next/src/server/base-server.ts | 14 +++--- packages/next/src/server/config-schema.ts | 1 + packages/next/src/server/config-shared.ts | 12 ++++++ packages/next/src/server/config.ts | 12 ++++++ .../next/src/server/lib/streaming-metadata.ts | 22 ++++++++++ packages/next/src/shared/lib/constants.ts | 1 + .../src/shared/lib/router/utils/is-bot.ts | 11 +++-- ...metadata-streaming-customized-rule.test.ts | 43 +++++++++++++++++++ .../metadata-streaming.test.ts | 2 +- .../metadata-streaming-config/app/layout.tsx | 8 ++++ .../metadata-streaming-config/app/page.tsx | 3 ++ ...tadata-streaming-config-customized.test.ts | 40 +++++++++++++++++ .../metadata-streaming-config.test.ts | 32 ++++++++++++++ .../metadata-streaming-config/next.config.js | 10 +++++ 18 files changed, 228 insertions(+), 12 deletions(-) create mode 100644 packages/next/src/server/lib/streaming-metadata.ts create mode 100644 test/e2e/app-dir/metadata-streaming/metadata-streaming-customized-rule.test.ts create mode 100644 test/production/app-dir/metadata-streaming-config/app/layout.tsx create mode 100644 test/production/app-dir/metadata-streaming-config/app/page.tsx create mode 100644 test/production/app-dir/metadata-streaming-config/metadata-streaming-config-customized.test.ts create mode 100644 test/production/app-dir/metadata-streaming-config/metadata-streaming-config.test.ts create mode 100644 test/production/app-dir/metadata-streaming-config/next.config.js diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 7c95b3d81daec..6f353ccc624e9 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -80,6 +80,7 @@ import { UNDERSCORE_NOT_FOUND_ROUTE_ENTRY, UNDERSCORE_NOT_FOUND_ROUTE, DYNAMIC_CSS_MANIFEST, + RESPONSE_CONFIG_MANIFEST, } from '../shared/lib/constants' import { getSortedRoutes, @@ -214,6 +215,7 @@ import { getParsedNodeOptionsWithoutInspect, } from '../server/lib/utils' import { InvariantError } from '../shared/lib/invariant-error' +import { HTML_LIMITED_BOT_UA_RE_STRING } from '../shared/lib/router/utils/is-bot' type Fallback = null | boolean | string @@ -1307,6 +1309,24 @@ export default async function build( NextBuildContext.clientRouterFilters = clientRouterFilters } + if (config.experimental.streamingMetadata) { + // Write html limited bots config to response-config-manifest + const responseConfigManifestPath = path.join( + distDir, + RESPONSE_CONFIG_MANIFEST + ) + const responseConfigManifest: { + version: number + htmlLimitedBots: string + } = { + version: 0, + htmlLimitedBots: + config.experimental.htmlLimitedBots || + HTML_LIMITED_BOT_UA_RE_STRING, + } + await writeManifest(responseConfigManifestPath, responseConfigManifest) + } + // Ensure commonjs handling is used for files in the distDir (generally .next) // Files outside of the distDir can be "type": "module" await writeFileUtf8( @@ -2487,6 +2507,9 @@ export default async function build( PRERENDER_MANIFEST, path.join(SERVER_DIRECTORY, MIDDLEWARE_MANIFEST), path.join(SERVER_DIRECTORY, MIDDLEWARE_BUILD_MANIFEST + '.js'), + ...(config.experimental.streamingMetadata + ? [RESPONSE_CONFIG_MANIFEST] + : []), ...(!process.env.TURBOPACK ? [ path.join( diff --git a/packages/next/src/build/utils.ts b/packages/next/src/build/utils.ts index 4a75c749fbe50..c032856b1cbf0 100644 --- a/packages/next/src/build/utils.ts +++ b/packages/next/src/build/utils.ts @@ -1,4 +1,4 @@ -import type { NextConfig, NextConfigComplete } from '../server/config-shared' +import type { NextConfigComplete } from '../server/config-shared' import type { ExperimentalPPRConfig } from '../server/lib/experimental/ppr' import type { AppBuildManifest } from './webpack/plugins/app-build-manifest-plugin' import type { AssetBinding } from './webpack/loaders/get-module-build-info' @@ -1466,7 +1466,7 @@ export async function copyTracedFiles( pageKeys: readonly string[], appPageKeys: readonly string[] | undefined, tracingRoot: string, - serverConfig: NextConfig, + serverConfig: NextConfigComplete, middlewareManifest: MiddlewareManifest, hasInstrumentationHook: boolean, staticPages: Set diff --git a/packages/next/src/export/index.ts b/packages/next/src/export/index.ts index b568b0ac5e8b6..e09daa276aa73 100644 --- a/packages/next/src/export/index.ts +++ b/packages/next/src/export/index.ts @@ -360,6 +360,7 @@ async function exportAppImpl( inlineCss: nextConfig.experimental.inlineCss ?? false, authInterrupts: !!nextConfig.experimental.authInterrupts, streamingMetadata: !!nextConfig.experimental.streamingMetadata, + htmlLimitedBots: nextConfig.experimental.htmlLimitedBots, }, reactMaxHeadersLength: nextConfig.reactMaxHeadersLength, } diff --git a/packages/next/src/server/app-render/types.ts b/packages/next/src/server/app-render/types.ts index 8046de212cd7b..221c471c30a0a 100644 --- a/packages/next/src/server/app-render/types.ts +++ b/packages/next/src/server/app-render/types.ts @@ -216,6 +216,7 @@ export interface RenderOptsPartial { inlineCss: boolean authInterrupts: boolean streamingMetadata: boolean + htmlLimitedBots: string | undefined } postponed?: string diff --git a/packages/next/src/server/base-server.ts b/packages/next/src/server/base-server.ts index 58b92495620d9..f9a0264344274 100644 --- a/packages/next/src/server/base-server.ts +++ b/packages/next/src/server/base-server.ts @@ -83,7 +83,7 @@ import { } from './lib/revalidate' import { execOnce } from '../shared/lib/utils' import { isBlockedPage } from './utils' -import { isBot, isHtmlLimitedBotUA } from '../shared/lib/router/utils/is-bot' +import { isBot } from '../shared/lib/router/utils/is-bot' import RenderResult from './render-result' import { removeTrailingSlash } from '../shared/lib/router/utils/remove-trailing-slash' import { denormalizePagePath } from '../shared/lib/page-path/denormalize-page-path' @@ -175,6 +175,7 @@ import type { RouteModule } from './route-modules/route-module' import { FallbackMode, parseFallbackField } from '../lib/fallback' import { toResponseCacheEntry } from './response-cache/utils' import { scheduleOnNextTick } from '../lib/scheduler' +import { shouldServeStreamingMetadata } from './lib/streaming-metadata' export type FindComponentsResult = { components: LoadComponentsReturnType @@ -595,6 +596,7 @@ export default abstract class Server< inlineCss: this.nextConfig.experimental.inlineCss ?? false, authInterrupts: !!this.nextConfig.experimental.authInterrupts, streamingMetadata: !!this.nextConfig.experimental.streamingMetadata, + htmlLimitedBots: this.nextConfig.experimental.htmlLimitedBots, }, onInstrumentationRequestError: this.instrumentationOnRequestError.bind(this), @@ -1675,11 +1677,13 @@ export default abstract class Server< renderOpts: { ...this.renderOpts, supportsDynamicResponse: !isBotRequest, - serveStreamingMetadata: - this.renderOpts.experimental.streamingMetadata && - !isHtmlLimitedBotUA(ua), + serveStreamingMetadata: shouldServeStreamingMetadata( + ua, + this.renderOpts.experimental + ), }, } + const payload = await fn(ctx) if (payload === null) { return @@ -2181,8 +2185,6 @@ export default abstract class Server< // cache if there are no dynamic data requirements opts.supportsDynamicResponse = !isSSG && !isBotRequest && !query.amp && isSupportedDocument - opts.serveStreamingMetadata = - opts.experimental.streamingMetadata && !isHtmlLimitedBotUA(ua) } // In development, we always want to generate dynamic HTML. diff --git a/packages/next/src/server/config-schema.ts b/packages/next/src/server/config-schema.ts index 26f28994a3bcd..a1aacdc2133b6 100644 --- a/packages/next/src/server/config-schema.ts +++ b/packages/next/src/server/config-schema.ts @@ -442,6 +442,7 @@ export const configSchema: zod.ZodType = z.lazy(() => authInterrupts: z.boolean().optional(), newDevOverlay: z.boolean().optional(), streamingMetadata: z.boolean().optional(), + htmlLimitedBots: z.instanceof(RegExp).optional(), }) .optional(), exportPathMap: z diff --git a/packages/next/src/server/config-shared.ts b/packages/next/src/server/config-shared.ts index 1f1c1e9e70aa3..77e2ca7ba53f3 100644 --- a/packages/next/src/server/config-shared.ts +++ b/packages/next/src/server/config-shared.ts @@ -21,6 +21,11 @@ export type NextConfigComplete = Required & { configOrigin?: string configFile?: string configFileName: string + // override NextConfigComplete.experimental.htmlLimitedBots to string + // because it's not defined in NextConfigComplete.experimental + experimental: Omit & { + htmlLimitedBots: string | undefined + } } export type I18NDomains = readonly DomainLocale[] @@ -577,6 +582,12 @@ export interface ExperimentalConfig { * When enabled will cause async metadata calls to stream rather than block the render. */ streamingMetadata?: boolean + + /** + * User Agent of bots that can handle streaming metadata. + * Besides the default behavior, Next.js act differently on serving metadata to bots based on their capability. + */ + htmlLimitedBots?: RegExp } export type ExportPathMap = { @@ -1203,6 +1214,7 @@ export const defaultConfig: NextConfig = { inlineCss: false, newDevOverlay: false, streamingMetadata: false, + htmlLimitedBots: undefined, }, bundlePagesRouterDependencies: false, } diff --git a/packages/next/src/server/config.ts b/packages/next/src/server/config.ts index aa8398586e667..d45d7fd767cf6 100644 --- a/packages/next/src/server/config.ts +++ b/packages/next/src/server/config.ts @@ -27,6 +27,7 @@ import { hasNextSupport } from '../server/ci-info' import { transpileConfig } from '../build/next-config-ts/transpile-config' import { dset } from '../shared/lib/dset' import { normalizeZodErrors } from '../shared/lib/zod' +import { HTML_LIMITED_BOT_UA_RE_STRING } from '../shared/lib/router/utils/is-bot' export { normalizeConfig } from './config-shared' export type { DomainLocale, NextConfig } from './config-shared' @@ -1004,6 +1005,11 @@ function assignDefaults( ]), ] + if (!result.experimental.htmlLimitedBots) { + // @ts-expect-error: override the htmlLimitedBots with default string, type covert: RegExp -> string + result.experimental.htmlLimitedBots = HTML_LIMITED_BOT_UA_RE_STRING + } + return result } @@ -1222,6 +1228,12 @@ export default async function loadConfig( } } + // serialize the regex config into string + if (userConfig.experimental?.htmlLimitedBots instanceof RegExp) { + userConfig.experimental.htmlLimitedBots = + userConfig.experimental.htmlLimitedBots.source + } + onLoadUserConfig?.(userConfig) const completeConfig = assignDefaults( dir, diff --git a/packages/next/src/server/lib/streaming-metadata.ts b/packages/next/src/server/lib/streaming-metadata.ts new file mode 100644 index 0000000000000..e4f3ce40a9e38 --- /dev/null +++ b/packages/next/src/server/lib/streaming-metadata.ts @@ -0,0 +1,22 @@ +import { HTML_LIMITED_BOT_UA_RE_STRING } from '../../shared/lib/router/utils/is-bot' + +export function shouldServeStreamingMetadata( + userAgent: string, + { + streamingMetadata, + htmlLimitedBots, + }: { + streamingMetadata: boolean + htmlLimitedBots: string | undefined + } +): boolean { + if (!streamingMetadata) { + return false + } + + const blockingMetadataUARegex = new RegExp( + htmlLimitedBots || HTML_LIMITED_BOT_UA_RE_STRING, + 'i' + ) + return !blockingMetadataUARegex.test(userAgent) +} diff --git a/packages/next/src/shared/lib/constants.ts b/packages/next/src/shared/lib/constants.ts index 82f1b064100c1..eb44dcc673604 100644 --- a/packages/next/src/shared/lib/constants.ts +++ b/packages/next/src/shared/lib/constants.ts @@ -42,6 +42,7 @@ export const EXPORT_DETAIL = 'export-detail.json' export const PRERENDER_MANIFEST = 'prerender-manifest.json' export const ROUTES_MANIFEST = 'routes-manifest.json' export const IMAGES_MANIFEST = 'images-manifest.json' +export const RESPONSE_CONFIG_MANIFEST = 'response-config-manifest.json' export const SERVER_FILES_MANIFEST = 'required-server-files.json' export const DEV_CLIENT_PAGES_MANIFEST = '_devPagesManifest.json' export const MIDDLEWARE_MANIFEST = 'middleware-manifest.json' diff --git a/packages/next/src/shared/lib/router/utils/is-bot.ts b/packages/next/src/shared/lib/router/utils/is-bot.ts index ca1e02d20215d..019a09f90024c 100644 --- a/packages/next/src/shared/lib/router/utils/is-bot.ts +++ b/packages/next/src/shared/lib/router/utils/is-bot.ts @@ -4,14 +4,19 @@ const HEADLESS_BROWSER_BOT_UA_RE = // This regex contains the bots that we need to do a blocking render for and can't safely stream the response // due to how they parse the DOM. For example, they might explicitly check for metadata in the `head` tag, so we can't stream metadata tags after the `head` was sent. -const HTML_LIMITED_BOT_UA_RE = - /Mediapartners-Google|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview/i +export const HTML_LIMITED_BOT_UA_RE_STRING = + 'Mediapartners-Google|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview' + +export const HTML_LIMITED_BOT_UA_RE = new RegExp( + HTML_LIMITED_BOT_UA_RE_STRING, + 'i' +) function isHeadlessBrowserBotUA(userAgent: string) { return HEADLESS_BROWSER_BOT_UA_RE.test(userAgent) } -export function isHtmlLimitedBotUA(userAgent: string) { +function isHtmlLimitedBotUA(userAgent: string) { return HTML_LIMITED_BOT_UA_RE.test(userAgent) } diff --git a/test/e2e/app-dir/metadata-streaming/metadata-streaming-customized-rule.test.ts b/test/e2e/app-dir/metadata-streaming/metadata-streaming-customized-rule.test.ts new file mode 100644 index 0000000000000..a188d924973a3 --- /dev/null +++ b/test/e2e/app-dir/metadata-streaming/metadata-streaming-customized-rule.test.ts @@ -0,0 +1,43 @@ +import { nextTestSetup } from 'e2e-utils' + +describe('app-dir - metadata-streaming-customized-rule', () => { + const { next } = nextTestSetup({ + files: __dirname, + overrideFiles: { + 'next.config.js': ` + module.exports = { + experimental: { + streamingMetadata: true, + htmlLimitedBots: /Minibot/i, + } + } + `, + }, + }) + + it('should send the blocking response for html limited bots', async () => { + const $ = await next.render$( + '/', + undefined, // no query + { + headers: { + 'user-agent': 'Minibot', + }, + } + ) + expect(await $('title').text()).toBe('index page') + }) + + it('should send streaming response for headless browser bots', async () => { + const $ = await next.render$( + '/', + undefined, // no query + { + headers: { + 'user-agent': 'Weebot', + }, + } + ) + expect(await $('title').length).toBe(0) + }) +}) diff --git a/test/e2e/app-dir/metadata-streaming/metadata-streaming.test.ts b/test/e2e/app-dir/metadata-streaming/metadata-streaming.test.ts index 0c3ee190f89cd..3c4ed76422618 100644 --- a/test/e2e/app-dir/metadata-streaming/metadata-streaming.test.ts +++ b/test/e2e/app-dir/metadata-streaming/metadata-streaming.test.ts @@ -1,7 +1,7 @@ import { nextTestSetup } from 'e2e-utils' import { retry, createMultiDomMatcher } from 'next-test-utils' -describe('metadata-streaming', () => { +describe('app-dir - metadata-streaming', () => { const { next } = nextTestSetup({ files: __dirname, }) diff --git a/test/production/app-dir/metadata-streaming-config/app/layout.tsx b/test/production/app-dir/metadata-streaming-config/app/layout.tsx new file mode 100644 index 0000000000000..888614deda3ba --- /dev/null +++ b/test/production/app-dir/metadata-streaming-config/app/layout.tsx @@ -0,0 +1,8 @@ +import { ReactNode } from 'react' +export default function Root({ children }: { children: ReactNode }) { + return ( + + {children} + + ) +} diff --git a/test/production/app-dir/metadata-streaming-config/app/page.tsx b/test/production/app-dir/metadata-streaming-config/app/page.tsx new file mode 100644 index 0000000000000..ff7159d9149fe --- /dev/null +++ b/test/production/app-dir/metadata-streaming-config/app/page.tsx @@ -0,0 +1,3 @@ +export default function Page() { + return

hello world

+} diff --git a/test/production/app-dir/metadata-streaming-config/metadata-streaming-config-customized.test.ts b/test/production/app-dir/metadata-streaming-config/metadata-streaming-config-customized.test.ts new file mode 100644 index 0000000000000..126dcaf90e46f --- /dev/null +++ b/test/production/app-dir/metadata-streaming-config/metadata-streaming-config-customized.test.ts @@ -0,0 +1,40 @@ +import { nextTestSetup } from 'e2e-utils' + +describe('app-dir - metadata-streaming-config-customized', () => { + const { next } = nextTestSetup({ + files: __dirname, + overrideFiles: { + 'next.config.js': ` + module.exports = { + experimental: { + streamingMetadata: true, + htmlLimitedBots: /MyBot/i, + } + } + `, + }, + }) + + it('should have the default streaming metadata config output in routes-manifest.json', async () => { + const requiredServerFiles = JSON.parse( + await next.readFile('.next/required-server-files.json') + ) + expect(requiredServerFiles.files).toContain( + '.next/response-config-manifest.json' + ) + expect( + requiredServerFiles.config.experimental.htmlLimitedBots + ).toMatchInlineSnapshot(`"MyBot"`) + + const responseConfigManifest = JSON.parse( + await next.readFile('.next/response-config-manifest.json') + ) + + expect(responseConfigManifest).toMatchInlineSnapshot(` + { + "htmlLimitedBots": "MyBot", + "version": 0, + } + `) + }) +}) diff --git a/test/production/app-dir/metadata-streaming-config/metadata-streaming-config.test.ts b/test/production/app-dir/metadata-streaming-config/metadata-streaming-config.test.ts new file mode 100644 index 0000000000000..ae24f713e56d4 --- /dev/null +++ b/test/production/app-dir/metadata-streaming-config/metadata-streaming-config.test.ts @@ -0,0 +1,32 @@ +import { nextTestSetup } from 'e2e-utils' + +describe('app-dir - metadata-streaming-config', () => { + const { next } = nextTestSetup({ + files: __dirname, + }) + + it('should have the default streaming metadata config output in routes-manifest.json', async () => { + const requiredServerFiles = JSON.parse( + await next.readFile('.next/required-server-files.json') + ) + expect(requiredServerFiles.files).toContain( + '.next/response-config-manifest.json' + ) + expect( + requiredServerFiles.config.experimental.htmlLimitedBots + ).toMatchInlineSnapshot( + `"Mediapartners-Google|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview"` + ) + + const responseConfigManifest = JSON.parse( + await next.readFile('.next/response-config-manifest.json') + ) + + expect(responseConfigManifest).toMatchInlineSnapshot(` + { + "htmlLimitedBots": "Mediapartners-Google|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview", + "version": 0, + } + `) + }) +}) diff --git a/test/production/app-dir/metadata-streaming-config/next.config.js b/test/production/app-dir/metadata-streaming-config/next.config.js new file mode 100644 index 0000000000000..427a5b2dcb098 --- /dev/null +++ b/test/production/app-dir/metadata-streaming-config/next.config.js @@ -0,0 +1,10 @@ +/** + * @type {import('next').NextConfig} + */ +const nextConfig = { + experimental: { + streamingMetadata: true, + }, +} + +module.exports = nextConfig From 3f044419502d006808131f830cb9341e99e28c09 Mon Sep 17 00:00:00 2001 From: vercel-release-bot Date: Fri, 10 Jan 2025 20:54:58 +0000 Subject: [PATCH 3/9] v15.2.0-canary.4 --- lerna.json | 2 +- packages/create-next-app/package.json | 2 +- packages/eslint-config-next/package.json | 4 ++-- packages/eslint-plugin-next/package.json | 2 +- packages/font/package.json | 2 +- packages/next-bundle-analyzer/package.json | 2 +- packages/next-codemod/package.json | 2 +- packages/next-env/package.json | 2 +- packages/next-mdx/package.json | 2 +- packages/next-plugin-storybook/package.json | 2 +- packages/next-polyfill-module/package.json | 2 +- packages/next-polyfill-nomodule/package.json | 2 +- packages/next-swc/package.json | 2 +- packages/next/package.json | 14 +++++++------- packages/react-refresh-utils/package.json | 2 +- packages/third-parties/package.json | 4 ++-- pnpm-lock.yaml | 16 ++++++++-------- 17 files changed, 32 insertions(+), 32 deletions(-) diff --git a/lerna.json b/lerna.json index 645a6ae6f0f66..df2297b29b8ce 100644 --- a/lerna.json +++ b/lerna.json @@ -16,5 +16,5 @@ "registry": "https://registry.npmjs.org/" } }, - "version": "15.2.0-canary.3" + "version": "15.2.0-canary.4" } diff --git a/packages/create-next-app/package.json b/packages/create-next-app/package.json index 0a53ee13a5392..955998de4b8f3 100644 --- a/packages/create-next-app/package.json +++ b/packages/create-next-app/package.json @@ -1,6 +1,6 @@ { "name": "create-next-app", - "version": "15.2.0-canary.3", + "version": "15.2.0-canary.4", "keywords": [ "react", "next", diff --git a/packages/eslint-config-next/package.json b/packages/eslint-config-next/package.json index 3967bac97f241..f4754c9bda604 100644 --- a/packages/eslint-config-next/package.json +++ b/packages/eslint-config-next/package.json @@ -1,6 +1,6 @@ { "name": "eslint-config-next", - "version": "15.2.0-canary.3", + "version": "15.2.0-canary.4", "description": "ESLint configuration used by Next.js.", "main": "index.js", "license": "MIT", @@ -10,7 +10,7 @@ }, "homepage": "https://nextjs.org/docs/app/api-reference/config/eslint", "dependencies": { - "@next/eslint-plugin-next": "15.2.0-canary.3", + "@next/eslint-plugin-next": "15.2.0-canary.4", "@rushstack/eslint-patch": "^1.10.3", "@typescript-eslint/eslint-plugin": "^5.4.2 || ^6.0.0 || ^7.0.0 || ^8.0.0", "@typescript-eslint/parser": "^5.4.2 || ^6.0.0 || ^7.0.0 || ^8.0.0", diff --git a/packages/eslint-plugin-next/package.json b/packages/eslint-plugin-next/package.json index 51d4bb48c591e..8e1ba11212d21 100644 --- a/packages/eslint-plugin-next/package.json +++ b/packages/eslint-plugin-next/package.json @@ -1,6 +1,6 @@ { "name": "@next/eslint-plugin-next", - "version": "15.2.0-canary.3", + "version": "15.2.0-canary.4", "description": "ESLint plugin for Next.js.", "main": "dist/index.js", "license": "MIT", diff --git a/packages/font/package.json b/packages/font/package.json index c560a496d9512..57e8ad628aead 100644 --- a/packages/font/package.json +++ b/packages/font/package.json @@ -1,7 +1,7 @@ { "name": "@next/font", "private": true, - "version": "15.2.0-canary.3", + "version": "15.2.0-canary.4", "repository": { "url": "vercel/next.js", "directory": "packages/font" diff --git a/packages/next-bundle-analyzer/package.json b/packages/next-bundle-analyzer/package.json index 6777c583fb9ac..2d015c9cc10b6 100644 --- a/packages/next-bundle-analyzer/package.json +++ b/packages/next-bundle-analyzer/package.json @@ -1,6 +1,6 @@ { "name": "@next/bundle-analyzer", - "version": "15.2.0-canary.3", + "version": "15.2.0-canary.4", "main": "index.js", "types": "index.d.ts", "license": "MIT", diff --git a/packages/next-codemod/package.json b/packages/next-codemod/package.json index 7d60e789e6944..4bcd6e7e9039c 100644 --- a/packages/next-codemod/package.json +++ b/packages/next-codemod/package.json @@ -1,6 +1,6 @@ { "name": "@next/codemod", - "version": "15.2.0-canary.3", + "version": "15.2.0-canary.4", "license": "MIT", "repository": { "type": "git", diff --git a/packages/next-env/package.json b/packages/next-env/package.json index 35a968a94518a..5d2b2b34fc8c4 100644 --- a/packages/next-env/package.json +++ b/packages/next-env/package.json @@ -1,6 +1,6 @@ { "name": "@next/env", - "version": "15.2.0-canary.3", + "version": "15.2.0-canary.4", "keywords": [ "react", "next", diff --git a/packages/next-mdx/package.json b/packages/next-mdx/package.json index 92a243d9a7742..e198246310f30 100644 --- a/packages/next-mdx/package.json +++ b/packages/next-mdx/package.json @@ -1,6 +1,6 @@ { "name": "@next/mdx", - "version": "15.2.0-canary.3", + "version": "15.2.0-canary.4", "main": "index.js", "license": "MIT", "repository": { diff --git a/packages/next-plugin-storybook/package.json b/packages/next-plugin-storybook/package.json index c6e050c963fe2..520ff3eb05073 100644 --- a/packages/next-plugin-storybook/package.json +++ b/packages/next-plugin-storybook/package.json @@ -1,6 +1,6 @@ { "name": "@next/plugin-storybook", - "version": "15.2.0-canary.3", + "version": "15.2.0-canary.4", "repository": { "url": "vercel/next.js", "directory": "packages/next-plugin-storybook" diff --git a/packages/next-polyfill-module/package.json b/packages/next-polyfill-module/package.json index f5b8f6ef1a6fc..bfc5be40ab580 100644 --- a/packages/next-polyfill-module/package.json +++ b/packages/next-polyfill-module/package.json @@ -1,6 +1,6 @@ { "name": "@next/polyfill-module", - "version": "15.2.0-canary.3", + "version": "15.2.0-canary.4", "description": "A standard library polyfill for ES Modules supporting browsers (Edge 16+, Firefox 60+, Chrome 61+, Safari 10.1+)", "main": "dist/polyfill-module.js", "license": "MIT", diff --git a/packages/next-polyfill-nomodule/package.json b/packages/next-polyfill-nomodule/package.json index 1d1d75d341934..5f6aba32a6416 100644 --- a/packages/next-polyfill-nomodule/package.json +++ b/packages/next-polyfill-nomodule/package.json @@ -1,6 +1,6 @@ { "name": "@next/polyfill-nomodule", - "version": "15.2.0-canary.3", + "version": "15.2.0-canary.4", "description": "A polyfill for non-dead, nomodule browsers.", "main": "dist/polyfill-nomodule.js", "license": "MIT", diff --git a/packages/next-swc/package.json b/packages/next-swc/package.json index b0357a4237e58..39a3aeebd8779 100644 --- a/packages/next-swc/package.json +++ b/packages/next-swc/package.json @@ -1,6 +1,6 @@ { "name": "@next/swc", - "version": "15.2.0-canary.3", + "version": "15.2.0-canary.4", "private": true, "scripts": { "clean": "node ../../scripts/rm.mjs native", diff --git a/packages/next/package.json b/packages/next/package.json index 8f69ba1e4bee9..ad8b6dafa1162 100644 --- a/packages/next/package.json +++ b/packages/next/package.json @@ -1,6 +1,6 @@ { "name": "next", - "version": "15.2.0-canary.3", + "version": "15.2.0-canary.4", "description": "The React Framework", "main": "./dist/server/next.js", "license": "MIT", @@ -99,7 +99,7 @@ ] }, "dependencies": { - "@next/env": "15.2.0-canary.3", + "@next/env": "15.2.0-canary.4", "@swc/counter": "0.1.3", "@swc/helpers": "0.5.15", "busboy": "1.6.0", @@ -164,11 +164,11 @@ "@jest/types": "29.5.0", "@mswjs/interceptors": "0.23.0", "@napi-rs/triples": "1.2.0", - "@next/font": "15.2.0-canary.3", - "@next/polyfill-module": "15.2.0-canary.3", - "@next/polyfill-nomodule": "15.2.0-canary.3", - "@next/react-refresh-utils": "15.2.0-canary.3", - "@next/swc": "15.2.0-canary.3", + "@next/font": "15.2.0-canary.4", + "@next/polyfill-module": "15.2.0-canary.4", + "@next/polyfill-nomodule": "15.2.0-canary.4", + "@next/react-refresh-utils": "15.2.0-canary.4", + "@next/swc": "15.2.0-canary.4", "@opentelemetry/api": "1.6.0", "@playwright/test": "1.41.2", "@storybook/addon-essentials": "^8.4.7", diff --git a/packages/react-refresh-utils/package.json b/packages/react-refresh-utils/package.json index f4e9f791cb7d0..c8d7e3dcb855b 100644 --- a/packages/react-refresh-utils/package.json +++ b/packages/react-refresh-utils/package.json @@ -1,6 +1,6 @@ { "name": "@next/react-refresh-utils", - "version": "15.2.0-canary.3", + "version": "15.2.0-canary.4", "description": "An experimental package providing utilities for React Refresh.", "repository": { "url": "vercel/next.js", diff --git a/packages/third-parties/package.json b/packages/third-parties/package.json index b48859d486760..50ca8648fa80c 100644 --- a/packages/third-parties/package.json +++ b/packages/third-parties/package.json @@ -1,6 +1,6 @@ { "name": "@next/third-parties", - "version": "15.2.0-canary.3", + "version": "15.2.0-canary.4", "repository": { "url": "vercel/next.js", "directory": "packages/third-parties" @@ -26,7 +26,7 @@ "third-party-capital": "1.0.20" }, "devDependencies": { - "next": "15.2.0-canary.3", + "next": "15.2.0-canary.4", "outdent": "0.8.0", "prettier": "2.5.1", "typescript": "5.7.2" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5b7959dea84bf..708995c7618bd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -793,7 +793,7 @@ importers: packages/eslint-config-next: dependencies: '@next/eslint-plugin-next': - specifier: 15.2.0-canary.3 + specifier: 15.2.0-canary.4 version: link:../eslint-plugin-next '@rushstack/eslint-patch': specifier: ^1.10.3 @@ -857,7 +857,7 @@ importers: packages/next: dependencies: '@next/env': - specifier: 15.2.0-canary.3 + specifier: 15.2.0-canary.4 version: link:../next-env '@swc/counter': specifier: 0.1.3 @@ -985,19 +985,19 @@ importers: specifier: 1.2.0 version: 1.2.0 '@next/font': - specifier: 15.2.0-canary.3 + specifier: 15.2.0-canary.4 version: link:../font '@next/polyfill-module': - specifier: 15.2.0-canary.3 + specifier: 15.2.0-canary.4 version: link:../next-polyfill-module '@next/polyfill-nomodule': - specifier: 15.2.0-canary.3 + specifier: 15.2.0-canary.4 version: link:../next-polyfill-nomodule '@next/react-refresh-utils': - specifier: 15.2.0-canary.3 + specifier: 15.2.0-canary.4 version: link:../react-refresh-utils '@next/swc': - specifier: 15.2.0-canary.3 + specifier: 15.2.0-canary.4 version: link:../next-swc '@opentelemetry/api': specifier: 1.6.0 @@ -1661,7 +1661,7 @@ importers: version: 1.0.20 devDependencies: next: - specifier: 15.2.0-canary.3 + specifier: 15.2.0-canary.4 version: link:../next outdent: specifier: 0.8.0 From d737cefcc41e9d2b96c460865706d05304cf5c2c Mon Sep 17 00:00:00 2001 From: Sam Ko Date: Fri, 10 Jan 2025 12:57:37 -0800 Subject: [PATCH 4/9] chore(github): ignore PRs in for stale issue bot (#74767) ## Why? We have to explicitly give a `-1` for `days-before-pr-close` and `days-before-pr-stale` in order to ignore sifting through PRs. - https://github.com/vercel/next.js/actions/runs/12716117689/job/35449724177#step:2:376 --- .github/workflows/issue_stale.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/issue_stale.yml b/.github/workflows/issue_stale.yml index e4e67f56a584b..5dadf57796354 100644 --- a/.github/workflows/issue_stale.yml +++ b/.github/workflows/issue_stale.yml @@ -18,6 +18,8 @@ jobs: ascending: true days-before-issue-stale: 730 # issues with no activity in over two years days-before-issue-close: 7 + days-before-pr-close: -1 + days-before-pr-stale: -1 remove-issue-stale-when-updated: true stale-issue-label: 'stale' stale-issue-message: 'This issue has been automatically marked as stale due to two years of inactivity. It will be closed in 7 days unless there’s further input. If you believe this issue is still relevant, please leave a comment or provide updated details. Thank you.' From c6ac0b9f6cfcc23859947c9a5edc274ec74b8a2a Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Fri, 10 Jan 2025 14:33:50 -0800 Subject: [PATCH 5/9] SingleModuleGraph: yield edge weights during traversal (#74620) This adds the edge weight (the `ChunkingType`) of the edge during `traverse_edges_from_entry` and `traverse_edges_from_entry_topological`, rather than only providing only the originating node. Test Plan: Added a log to verify edge types were provided to these visitors. --- crates/next-api/src/module_graph.rs | 12 +-- .../turbopack-core/src/module_graph/mod.rs | 98 ++++++++++++++----- 2 files changed, 78 insertions(+), 32 deletions(-) diff --git a/crates/next-api/src/module_graph.rs b/crates/next-api/src/module_graph.rs index b446f74516f7b..d96f227e286ff 100644 --- a/crates/next-api/src/module_graph.rs +++ b/crates/next-api/src/module_graph.rs @@ -174,9 +174,9 @@ impl NextDynamicGraph { // module -> the client reference entry (if any) let mut state_map = HashMap::new(); - graph.traverse_edges_from_entry(entry, |(parent_node, node)| { + graph.traverse_edges_from_entry(entry, |parent_info, node| { let module = node.module; - let Some(parent_node) = parent_node else { + let Some((parent_node, _)) = parent_info else { state_map.insert(module, VisitState::Entry); return GraphTraversalAction::Continue; }; @@ -362,9 +362,9 @@ impl ClientReferencesGraph { entry, // state_map is `module -> Option< the current so parent server component >` &mut HashMap::new(), - |(parent_node, node), state_map| { + |parent_info, node, state_map| { let module = node.module; - let Some(parent_node) = parent_node else { + let Some((parent_node, _)) = parent_info else { state_map.insert(module, None); return GraphTraversalAction::Continue; }; @@ -390,8 +390,8 @@ impl ClientReferencesGraph { _ => GraphTraversalAction::Continue, } }, - |(parent_node, node), state_map| { - let Some(parent_node) = parent_node else { + |parent_info, node, state_map| { + let Some((parent_node, _)) = parent_info else { return; }; let parent_module = parent_node.module; diff --git a/turbopack/crates/turbopack-core/src/module_graph/mod.rs b/turbopack/crates/turbopack-core/src/module_graph/mod.rs index 7f49434075e2c..5f812eeb120e7 100644 --- a/turbopack/crates/turbopack-core/src/module_graph/mod.rs +++ b/turbopack/crates/turbopack-core/src/module_graph/mod.rs @@ -6,7 +6,7 @@ use std::{ use anyhow::{Context, Result}; use petgraph::{ - graph::{DiGraph, NodeIndex}, + graph::{DiGraph, EdgeIndex, NodeIndex}, visit::{Dfs, VisitMap, Visitable}, }; use serde::{Deserialize, Serialize}; @@ -212,11 +212,18 @@ impl SingleModuleGraph { /// target. /// /// This means that target nodes can be revisited (once per incoming edge). + /// + /// * `entry` - The entry module to start the traversal from + /// * `visitor` - Called before visiting the children of a node. + /// - Receives (originating &SingleModuleGraphNode, edge &ChunkingType), target + /// &SingleModuleGraphNode, state &S + /// - Can return [GraphTraversalAction]s to control the traversal pub fn traverse_edges_from_entry<'a>( &'a self, entry: ResolvedVc>, mut visitor: impl FnMut( - (Option<&'a SingleModuleGraphNode>, &'a SingleModuleGraphNode), + Option<(&'a SingleModuleGraphNode, &'a ChunkingType)>, + &'a SingleModuleGraphNode, ) -> GraphTraversalAction, ) -> Result<()> { let graph = &self.graph; @@ -226,14 +233,24 @@ impl SingleModuleGraph { let mut discovered = graph.visit_map(); let entry_weight = graph.node_weight(entry_node).unwrap(); entry_weight.emit_issues(); - visitor((None, entry_weight)); + visitor(None, entry_weight); while let Some(node) = stack.pop() { let node_weight = graph.node_weight(node).unwrap(); if discovered.visit(node) { - for succ in graph.neighbors(node).collect::>() { + let neighbors = { + let mut neighbors = vec![]; + let mut walker = graph.neighbors(node).detach(); + while let Some((edge, succ)) = walker.next(graph) { + neighbors.push((edge, succ)); + } + neighbors + }; + + for (edge, succ) in neighbors { let succ_weight = graph.node_weight(succ).unwrap(); - let action = visitor((Some(node_weight), succ_weight)); + let edge_weight = graph.edge_weight(edge).unwrap(); + let action = visitor(Some((node_weight, edge_weight)), succ_weight); if !discovered.is_visited(&succ) && action == GraphTraversalAction::Continue { stack.push(succ); } @@ -251,16 +268,29 @@ impl SingleModuleGraph { /// /// Target nodes can be revisited (once per incoming edge). /// Edges are traversed in normal order, so should correspond to reference order. + /// + /// * `entry` - The entry module to start the traversal from + /// * `state` - The state to be passed to the visitors + /// * `visit_preorder` - Called before visiting the children of a node. + /// - Receives: (originating &SingleModuleGraphNode, edge &ChunkingType), target + /// &SingleModuleGraphNode, state &S + /// - Can return [GraphTraversalAction]s to control the traversal + /// * `visit_postorder` - Called after visiting the children of a node. Return + /// - Receives: (originating &SingleModuleGraphNode, edge &ChunkingType), target + /// &SingleModuleGraphNode, state &S + /// - Can return [GraphTraversalAction]s to control the traversal pub fn traverse_edges_from_entry_topological<'a, S>( &'a self, entry: ResolvedVc>, state: &mut S, mut visit_preorder: impl FnMut( - (Option<&'a SingleModuleGraphNode>, &'a SingleModuleGraphNode), + Option<(&'a SingleModuleGraphNode, &'a ChunkingType)>, + &'a SingleModuleGraphNode, &mut S, ) -> GraphTraversalAction, mut visit_postorder: impl FnMut( - (Option<&'a SingleModuleGraphNode>, &'a SingleModuleGraphNode), + Option<(&'a SingleModuleGraphNode, &'a ChunkingType)>, + &'a SingleModuleGraphNode, &mut S, ), ) -> Result<()> { @@ -272,39 +302,47 @@ impl SingleModuleGraph { ExpandAndVisit, } - let mut stack: Vec<(ReverseTopologicalPass, Option, NodeIndex)> = - vec![(ReverseTopologicalPass::ExpandAndVisit, None, entry_node)]; + #[allow(clippy::type_complexity)] // This is a temporary internal structure + let mut stack: Vec<( + ReverseTopologicalPass, + Option<(NodeIndex, EdgeIndex)>, + NodeIndex, + )> = vec![(ReverseTopologicalPass::ExpandAndVisit, None, entry_node)]; let mut expanded = HashSet::new(); while let Some((pass, parent, current)) = stack.pop() { match pass { ReverseTopologicalPass::Visit => { visit_postorder( - ( - parent.map(|parent| graph.node_weight(parent).unwrap()), - graph.node_weight(current).unwrap(), - ), + parent.map(|parent| { + ( + graph.node_weight(parent.0).unwrap(), + graph.edge_weight(parent.1).unwrap(), + ) + }), + graph.node_weight(current).unwrap(), state, ); } ReverseTopologicalPass::ExpandAndVisit => { let action = visit_preorder( - ( - parent.map(|parent| graph.node_weight(parent).unwrap()), - graph.node_weight(current).unwrap(), - ), + parent.map(|parent| { + ( + graph.node_weight(parent.0).unwrap(), + graph.edge_weight(parent.1).unwrap(), + ) + }), + graph.node_weight(current).unwrap(), state, ); stack.push((ReverseTopologicalPass::Visit, parent, current)); if expanded.insert(current) && action == GraphTraversalAction::Continue { - stack.extend( - graph - .neighbors(current) - // .collect::>() - // .rev() - .map(|child| { - (ReverseTopologicalPass::ExpandAndVisit, Some(current), child) - }), - ); + stack.extend(iter_neighbors(graph, current).map(|(edge, child)| { + ( + ReverseTopologicalPass::ExpandAndVisit, + Some((current, edge)), + child, + ) + })); } } } @@ -533,3 +571,11 @@ impl Visit for SingleModuleGraphBuilder { } } } + +fn iter_neighbors( + graph: &DiGraph, + node: NodeIndex, +) -> impl Iterator + '_ { + let mut walker = graph.neighbors(node).detach(); + std::iter::from_fn(move || walker.next(graph)) +} From 40a4c3b7bdcd9d1fa1726dbd7c91c63be81cd665 Mon Sep 17 00:00:00 2001 From: Benjamin Woodruff Date: Fri, 10 Jan 2025 14:39:34 -0800 Subject: [PATCH 6/9] refactor(turbopack/next-api): Implement NonLocalValue for TracedDiGraph and SingleModuleGraph (#74506) This allows `SingleModuleGraph` to implement `NonLocalValue`, removing the 'local' opt-out on the `turbo_tasks::value` macro. --- .../turbopack-core/src/module_graph/mod.rs | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/turbopack/crates/turbopack-core/src/module_graph/mod.rs b/turbopack/crates/turbopack-core/src/module_graph/mod.rs index 5f812eeb120e7..d4d5953cc9ecd 100644 --- a/turbopack/crates/turbopack-core/src/module_graph/mod.rs +++ b/turbopack/crates/turbopack-core/src/module_graph/mod.rs @@ -29,7 +29,7 @@ use crate::{ #[derive(Clone, Debug)] pub struct ModuleSet(pub HashSet>>); -#[turbo_tasks::value(cell = "new", eq = "manual", into = "new", local)] +#[turbo_tasks::value(cell = "new", eq = "manual", into = "new")] #[derive(Clone, Default)] pub struct SingleModuleGraph { graph: TracedDiGraph, @@ -387,13 +387,18 @@ impl SingleModuleGraphNode { } #[derive(Clone, Debug, ValueDebugFormat, Serialize, Deserialize)] -struct TracedDiGraph(DiGraph); -impl Default for TracedDiGraph { +struct TracedDiGraph(DiGraph); +impl Default for TracedDiGraph { fn default() -> Self { Self(Default::default()) } } -impl TraceRawVcs for TracedDiGraph { + +impl TraceRawVcs for TracedDiGraph +where + N: TraceRawVcs, + E: TraceRawVcs, +{ fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) { for node in self.0.node_weights() { node.trace_raw_vcs(trace_context); @@ -403,13 +408,21 @@ impl TraceRawVcs for TracedDiGraph { } } } -impl Deref for TracedDiGraph { + +impl Deref for TracedDiGraph { type Target = DiGraph; fn deref(&self) -> &Self::Target { &self.0 } } +unsafe impl NonLocalValue for TracedDiGraph +where + N: NonLocalValue, + E: NonLocalValue, +{ +} + #[derive(PartialEq, Eq, Debug)] pub enum GraphTraversalAction { /// Continue visiting children From 4ef926bca5196673908b8efe9259034c1a6ef876 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Fri, 10 Jan 2025 15:21:58 -0800 Subject: [PATCH 7/9] Turbopack chunking: use ChunkableModules in `chunk_content`, not ChunkItems (#74040) This: - Replaces all use of chunk items in the interface for `chunk_content` with ChunkableModules - Moves `is_self_async` from `ChunkItem` to `Module` - Renames `AvailableChunkItems` to `AvailableModules` and all associated structs and methods, using modules in place of chunk items --------- Co-authored-by: Tobias Koppers --- crates/next-api/src/server_actions.rs | 2 +- .../client_reference_manifest.rs | 52 +-- .../turbopack-browser/src/chunking_context.rs | 6 +- .../src/ecmascript/content_entry.rs | 30 +- turbopack/crates/turbopack-browser/src/lib.rs | 1 + .../src/chunk/availability_info.rs | 26 +- .../src/chunk/available_modules.rs | 101 ++++++ .../turbopack-core/src/chunk/chunk_group.rs | 134 ++++--- .../turbopack-core/src/chunk/chunking.rs | 86 +++-- .../crates/turbopack-core/src/chunk/mod.rs | 338 ++++++++---------- turbopack/crates/turbopack-core/src/module.rs | 5 + .../crates/turbopack-css/src/chunk/mod.rs | 13 +- turbopack/crates/turbopack-css/src/lib.rs | 1 + .../src/async_chunk/chunk_item.rs | 17 +- .../src/chunk/chunk_type.rs | 39 +- .../turbopack-ecmascript/src/chunk/content.rs | 36 +- .../turbopack-ecmascript/src/chunk/item.rs | 15 +- .../turbopack-ecmascript/src/chunk/mod.rs | 46 +-- .../crates/turbopack-ecmascript/src/lib.rs | 25 +- .../src/manifest/chunk_asset.rs | 16 +- .../src/references/async_module.rs | 16 +- .../src/references/external_module.rs | 14 +- .../facade/chunk_item.rs | 24 +- .../side_effect_optimization/facade/module.rs | 13 + .../locals/chunk_item.rs | 12 - .../side_effect_optimization/locals/module.rs | 11 + .../src/tree_shake/asset.rs | 5 + .../src/tree_shake/chunk_item.rs | 5 - .../src/ecmascript/node/content.rs | 20 +- turbopack/crates/turbopack-nodejs/src/lib.rs | 1 + .../crates/turbopack-wasm/src/module_asset.rs | 10 +- 31 files changed, 652 insertions(+), 468 deletions(-) create mode 100644 turbopack/crates/turbopack-core/src/chunk/available_modules.rs diff --git a/crates/next-api/src/server_actions.rs b/crates/next-api/src/server_actions.rs index d928b58d8d47b..26f4feb5925db 100644 --- a/crates/next-api/src/server_actions.rs +++ b/crates/next-api/src/server_actions.rs @@ -166,7 +166,7 @@ async fn build_manifest( &key, ActionManifestWorkerEntry { module_id: ActionManifestModuleId::String(loader_id.as_str()), - is_async: *chunk_item.is_self_async().await?, + is_async: *chunk_item.module().is_self_async().await?, }, ); entry.layer.insert(&key, *layer); diff --git a/crates/next-core/src/next_manifests/client_reference_manifest.rs b/crates/next-core/src/next_manifests/client_reference_manifest.rs index 87697c80e89ca..ecb7f1c884e32 100644 --- a/crates/next-core/src/next_manifests/client_reference_manifest.rs +++ b/crates/next-core/src/next_manifests/client_reference_manifest.rs @@ -1,4 +1,4 @@ -use anyhow::Result; +use anyhow::{Context, Result}; use indoc::formatdoc; use turbo_rcstr::RcStr; use turbo_tasks::{FxIndexSet, ResolvedVc, TryJoinIterExt, Value, ValueToString, Vc}; @@ -6,8 +6,8 @@ use turbo_tasks_fs::{File, FileSystemPath}; use turbopack_core::{ asset::{Asset, AssetContent}, chunk::{ - availability_info::AvailabilityInfo, ChunkItem, ChunkItemExt, ChunkableModule, - ChunkingContext, ModuleId as TurbopackModuleId, + availability_info::AvailabilityInfo, ChunkItemExt, ChunkableModule, ChunkingContext, + ModuleId as TurbopackModuleId, }, output::{OutputAsset, OutputAssets}, virtual_output::VirtualOutputAsset, @@ -75,8 +75,8 @@ impl ClientReferenceManifest { let server_path = ecmascript_client_reference.server_ident.to_string().await?; - let client_chunk_item = ecmascript_client_reference - .client_module + let client_module = ecmascript_client_reference.client_module; + let client_chunk_item = client_module .as_chunk_item(Vc::upcast(client_chunking_context)) .to_resolved() .await?; @@ -107,8 +107,11 @@ impl ClientReferenceManifest { .map(RcStr::from) .collect::>(); - let is_async = - is_item_async(client_availability_info, client_chunk_item).await?; + let is_async = is_item_async( + client_availability_info, + ResolvedVc::upcast(client_module), + ) + .await?; (chunk_paths, is_async) } else { @@ -116,19 +119,20 @@ impl ClientReferenceManifest { }; if let Some(ssr_chunking_context) = ssr_chunking_context { - let ssr_chunk_item = ecmascript_client_reference - .ssr_module + let ssr_module = ecmascript_client_reference.ssr_module; + let ssr_chunk_item = ssr_module .as_chunk_item(Vc::upcast(ssr_chunking_context)) .to_resolved() .await?; let ssr_module_id = ssr_chunk_item.id().await?; - let rsc_chunk_item: ResolvedVc> = - ResolvedVc::try_downcast_type::( - parent_module, - ) - .await? - .unwrap() + let rsc_module = ResolvedVc::try_downcast_type::< + EcmascriptClientReferenceProxyModule, + >(parent_module) + .await? + .context("Expected EcmascriptClientReferenceProxyModule")?; + + let rsc_chunk_item = rsc_module .as_chunk_item(Vc::upcast(ssr_chunking_context)) .to_resolved() .await?; @@ -161,7 +165,9 @@ impl ClientReferenceManifest { .map(RcStr::from) .collect::>(); - let is_async = is_item_async(ssr_availability_info, ssr_chunk_item).await?; + let is_async = + is_item_async(ssr_availability_info, ResolvedVc::upcast(ssr_module)) + .await?; (chunk_paths, is_async) } else { @@ -188,9 +194,11 @@ impl ClientReferenceManifest { .map(RcStr::from) .collect::>(); - let is_async = - is_item_async(&rsc_app_entry_chunks_availability, rsc_chunk_item) - .await?; + let is_async = is_item_async( + &rsc_app_entry_chunks_availability, + ResolvedVc::upcast(rsc_module), + ) + .await?; (chunk_paths, is_async) }; @@ -368,13 +376,13 @@ pub fn get_client_reference_module_key(server_path: &str, export_name: &str) -> async fn is_item_async( availability_info: &AvailabilityInfo, - chunk_item: ResolvedVc>, + module: ResolvedVc>, ) -> Result { - let Some(available_chunk_items) = availability_info.available_chunk_items() else { + let Some(available_modules) = availability_info.available_modules() else { return Ok(false); }; - let Some(info) = available_chunk_items.await?.get(chunk_item).await? else { + let Some(info) = &*available_modules.get(*module).await? else { return Ok(false); }; diff --git a/turbopack/crates/turbopack-browser/src/chunking_context.rs b/turbopack/crates/turbopack-browser/src/chunking_context.rs index 60226f7522a43..5ce366e04052e 100644 --- a/turbopack/crates/turbopack-browser/src/chunking_context.rs +++ b/turbopack/crates/turbopack-browser/src/chunking_context.rs @@ -422,11 +422,9 @@ impl ChunkingContext for BrowserChunkingContext { AvailabilityInfo::Untracked => { ident = ident.with_modifier(Vc::cell("untracked".into())); } - AvailabilityInfo::Complete { - available_chunk_items, - } => { + AvailabilityInfo::Complete { available_modules } => { ident = ident.with_modifier(Vc::cell( - available_chunk_items.hash().await?.to_string().into(), + available_modules.hash().await?.to_string().into(), )); } } diff --git a/turbopack/crates/turbopack-browser/src/ecmascript/content_entry.rs b/turbopack/crates/turbopack-browser/src/ecmascript/content_entry.rs index 8c3d01e151437..cc80ad25780d7 100644 --- a/turbopack/crates/turbopack-browser/src/ecmascript/content_entry.rs +++ b/turbopack/crates/turbopack-browser/src/ecmascript/content_entry.rs @@ -4,13 +4,14 @@ use anyhow::Result; use tracing::{info_span, Instrument}; use turbo_tasks::{FxIndexMap, ReadRef, ResolvedVc, TryJoinIterExt, ValueToString, Vc}; use turbopack_core::{ - chunk::{AsyncModuleInfo, ChunkItem, ChunkItemExt, ModuleId}, + chunk::{AsyncModuleInfo, ChunkItem, ChunkItemExt, ChunkItemTy, ModuleId}, code_builder::{Code, CodeBuilder}, error::PrettyPrintError, issue::{code_gen::CodeGenerationIssue, IssueExt, IssueSeverity, StyledString}, }; use turbopack_ecmascript::chunk::{ EcmascriptChunkContent, EcmascriptChunkItem, EcmascriptChunkItemExt, + EcmascriptChunkItemWithAsyncInfo, }; /// A chunk item's content entry. @@ -28,7 +29,7 @@ pub struct EcmascriptDevChunkContentEntry { impl EcmascriptDevChunkContentEntry { pub async fn new( - chunk_item: Vc>, + chunk_item: ResolvedVc>, async_module_info: Option>, ) -> Result { let code = chunk_item.code(async_module_info).to_resolved().await?; @@ -52,15 +53,34 @@ impl EcmascriptDevChunkContentEntries { ) -> Result> { let chunk_content = chunk_content.await?; - let entries: FxIndexMap<_, _> = chunk_content + let included_chunk_items = chunk_content .chunk_items .iter() - .map(|&(chunk_item, async_module_info)| async move { + .map( + async |EcmascriptChunkItemWithAsyncInfo { + ty, + chunk_item, + async_info, + }| { + if matches!(ty, ChunkItemTy::Included) { + Ok(Some((chunk_item, async_info))) + } else { + Ok(None) + } + }, + ) + .try_join() + .await? + .into_iter() + .flatten(); + + let entries: FxIndexMap<_, _> = included_chunk_items + .map(|(&chunk_item, &async_module_info)| async move { async move { Ok(( chunk_item.id().await?, EcmascriptDevChunkContentEntry::new( - *chunk_item, + chunk_item, async_module_info.map(|info| *info), ) .await?, diff --git a/turbopack/crates/turbopack-browser/src/lib.rs b/turbopack/crates/turbopack-browser/src/lib.rs index 5c2c989539331..c5e14dfbcc7b8 100644 --- a/turbopack/crates/turbopack-browser/src/lib.rs +++ b/turbopack/crates/turbopack-browser/src/lib.rs @@ -1,3 +1,4 @@ +#![feature(async_closure)] #![feature(iter_intersperse)] #![feature(int_roundings)] #![feature(arbitrary_self_types)] diff --git a/turbopack/crates/turbopack-core/src/chunk/availability_info.rs b/turbopack/crates/turbopack-core/src/chunk/availability_info.rs index 5c90ca87d299d..3c3f4ea284c4d 100644 --- a/turbopack/crates/turbopack-core/src/chunk/availability_info.rs +++ b/turbopack/crates/turbopack-core/src/chunk/availability_info.rs @@ -1,7 +1,7 @@ use anyhow::Result; use turbo_tasks::{ResolvedVc, Vc}; -use super::available_chunk_items::{AvailableChunkItemInfoMap, AvailableChunkItems}; +use super::available_modules::{AvailableModuleInfoMap, AvailableModules}; #[turbo_tasks::value(serialization = "auto_for_input")] #[derive(Hash, Clone, Copy, Debug)] @@ -12,36 +12,30 @@ pub enum AvailabilityInfo { Root, /// There are modules already available. Complete { - available_chunk_items: ResolvedVc, + available_modules: ResolvedVc, }, } impl AvailabilityInfo { - pub fn available_chunk_items(&self) -> Option> { + pub fn available_modules(&self) -> Option> { match self { Self::Untracked => None, Self::Root => None, Self::Complete { - available_chunk_items, - .. - } => Some(*available_chunk_items), + available_modules, .. + } => Some(*available_modules), } } - pub async fn with_chunk_items( - self, - chunk_items: Vc, - ) -> Result { + pub async fn with_modules(self, modules: Vc) -> Result { Ok(match self { AvailabilityInfo::Untracked => AvailabilityInfo::Untracked, AvailabilityInfo::Root => AvailabilityInfo::Complete { - available_chunk_items: AvailableChunkItems::new(chunk_items).to_resolved().await?, + available_modules: AvailableModules::new(modules).to_resolved().await?, }, - AvailabilityInfo::Complete { - available_chunk_items, - } => AvailabilityInfo::Complete { - available_chunk_items: available_chunk_items - .with_chunk_items(chunk_items) + AvailabilityInfo::Complete { available_modules } => AvailabilityInfo::Complete { + available_modules: available_modules + .with_modules(modules) .to_resolved() .await?, }, diff --git a/turbopack/crates/turbopack-core/src/chunk/available_modules.rs b/turbopack/crates/turbopack-core/src/chunk/available_modules.rs new file mode 100644 index 0000000000000..d133ff0ac6c3d --- /dev/null +++ b/turbopack/crates/turbopack-core/src/chunk/available_modules.rs @@ -0,0 +1,101 @@ +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use turbo_tasks::{ + debug::ValueDebugFormat, trace::TraceRawVcs, FxIndexMap, NonLocalValue, ResolvedVc, + TryFlatJoinIterExt, TryJoinIterExt, ValueToString, Vc, +}; +use turbo_tasks_hash::Xxh3Hash64Hasher; + +use super::ChunkableModule; +use crate::module::Module; + +#[derive( + PartialEq, Eq, TraceRawVcs, Copy, Clone, Serialize, Deserialize, ValueDebugFormat, NonLocalValue, +)] +pub struct AvailableModulesInfo { + pub is_async: bool, +} + +#[turbo_tasks::value(transparent)] +pub struct OptionAvailableModulesInfo(Option); + +#[turbo_tasks::value(transparent)] +pub struct AvailableModuleInfoMap( + FxIndexMap>, AvailableModulesInfo>, +); + +/// Allows to gather information about which assets are already available. +/// Adding more roots will form a linked list like structure to allow caching +/// `include` queries. +#[turbo_tasks::value] +pub struct AvailableModules { + parent: Option>, + modules: ResolvedVc, +} + +#[turbo_tasks::value_impl] +impl AvailableModules { + #[turbo_tasks::function] + pub fn new(modules: ResolvedVc) -> Vc { + AvailableModules { + parent: None, + modules, + } + .cell() + } + + #[turbo_tasks::function] + pub async fn with_modules( + self: ResolvedVc, + modules: ResolvedVc, + ) -> Result> { + let modules = modules + .await? + .into_iter() + .map(|(&module, &info)| async move { + Ok(self.get(*module).await?.is_none().then_some((module, info))) + }) + .try_flat_join() + .await?; + Ok(AvailableModules { + parent: Some(self), + modules: ResolvedVc::cell(modules.into_iter().collect()), + } + .cell()) + } + + #[turbo_tasks::function] + pub async fn hash(&self) -> Result> { + let mut hasher = Xxh3Hash64Hasher::new(); + if let Some(parent) = self.parent { + hasher.write_value(parent.hash().await?); + } else { + hasher.write_value(0u64); + } + let item_idents = self + .modules + .await? + .iter() + .map(|(&module, _)| module.ident().to_string()) + .try_join() + .await?; + for ident in item_idents { + hasher.write_value(ident); + } + Ok(Vc::cell(hasher.finish())) + } + + #[turbo_tasks::function] + pub async fn get( + &self, + module: ResolvedVc>, + ) -> Result> { + if let Some(&info) = self.modules.await?.get(&module) { + return Ok(Vc::cell(Some(info))); + }; + if let Some(parent) = self.parent { + return Ok(parent.get(*module)); + } + Ok(Vc::cell(None)) + } +} diff --git a/turbopack/crates/turbopack-core/src/chunk/chunk_group.rs b/turbopack/crates/turbopack-core/src/chunk/chunk_group.rs index c83de772181aa..948baf23f4e31 100644 --- a/turbopack/crates/turbopack-core/src/chunk/chunk_group.rs +++ b/turbopack/crates/turbopack-core/src/chunk/chunk_group.rs @@ -2,15 +2,17 @@ use std::collections::HashSet; use anyhow::Result; use auto_hash_map::AutoSet; +use futures::future::Either; use turbo_tasks::{FxIndexMap, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Value, Vc}; use super::{ - availability_info::AvailabilityInfo, available_chunk_items::AvailableChunkItemInfo, - chunk_content, chunking::make_chunks, AsyncModuleInfo, Chunk, ChunkContentResult, ChunkItem, - ChunkingContext, + availability_info::AvailabilityInfo, available_modules::AvailableModulesInfo, chunk_content, + chunking::make_chunks, AsyncModuleInfo, Chunk, ChunkContentResult, ChunkItem, ChunkItemTy, + ChunkItemWithAsyncModuleInfo, ChunkableModule, ChunkingContext, }; use crate::{ - module::Module, output::OutputAssets, rebase::RebasedAsset, reference::ModuleReference, + environment::ChunkLoading, module::Module, output::OutputAssets, rebase::RebasedAsset, + reference::ModuleReference, }; pub struct MakeChunkGroupResult { @@ -24,40 +26,51 @@ pub async fn make_chunk_group( chunk_group_entries: impl IntoIterator>>, availability_info: AvailabilityInfo, ) -> Result { + let can_split_async = !matches!( + *chunking_context.environment().chunk_loading().await?, + ChunkLoading::Edge + ); + let should_trace = *chunking_context.is_tracing_enabled().await?; + let ChunkContentResult { - chunk_items, - external_output_assets, - external_module_references, + chunkable_modules, async_modules, traced_modules, + passthrough_modules, forward_edges_inherit_async, local_back_edges_inherit_async, available_async_modules_back_edges_inherit_async, - } = chunk_content(chunking_context, chunk_group_entries, availability_info).await?; + } = chunk_content( + chunk_group_entries, + availability_info, + can_split_async, + should_trace, + ) + .await?; // Find all local chunk items that are self async - let self_async_children = chunk_items + let self_async_children = chunkable_modules .iter() .copied() - .map(|chunk_item| async move { - let is_self_async = *chunk_item.is_self_async().await?; - Ok(is_self_async.then_some(chunk_item)) + .map(|m| async move { + let is_self_async = *m.is_self_async().await?; + Ok(is_self_async.then_some(m)) }) .try_flat_join() .await?; // Get all available async modules and concatenate with local async modules - let mut async_chunk_items = available_async_modules_back_edges_inherit_async + let mut all_async_modules = available_async_modules_back_edges_inherit_async .keys() .copied() .chain(self_async_children.into_iter()) - .map(|chunk_item| (chunk_item, AutoSet::>>::new())) + .map(|m| (m, AutoSet::>>::new())) .collect::>(); // Propagate async inheritance let mut i = 0; loop { - let Some((&chunk_item, _)) = async_chunk_items.get_index(i) else { + let Some((&async_module, _)) = all_async_modules.get_index(i) else { break; }; // The first few entries are from @@ -68,26 +81,26 @@ pub async fn make_chunk_group( } else { &local_back_edges_inherit_async }; - if let Some(parents) = map.get(&chunk_item) { + if let Some(parents) = map.get(&async_module) { for &parent in parents.iter() { // Add item, it will be iterated by this loop too - async_chunk_items + all_async_modules .entry(parent) .or_default() - .insert(chunk_item); + .insert(async_module); } } i += 1; } // Create map for chunk items with empty [Option>] - let mut chunk_items = chunk_items + let mut all_modules = chunkable_modules .into_iter() - .map(|chunk_item| (chunk_item, None)) + .map(|m| (m, None)) .collect::>>>(); // Insert AsyncModuleInfo for every async module - for (async_item, referenced_async_modules) in async_chunk_items { + for (async_item, referenced_async_modules) in all_async_modules { let referenced_async_modules = if let Some(references) = forward_edges_inherit_async.get(&async_item) { references @@ -99,7 +112,7 @@ pub async fn make_chunk_group( } else { Default::default() }; - chunk_items.insert( + all_modules.insert( async_item, Some( AsyncModuleInfo::new(referenced_async_modules) @@ -111,12 +124,12 @@ pub async fn make_chunk_group( // Compute new [AvailabilityInfo] let availability_info = { - let map = chunk_items + let map = all_modules .iter() - .map(|(&chunk_item, async_info)| async move { + .map(|(&module, async_info)| async move { Ok(( - chunk_item, - AvailableChunkItemInfo { + module, + AvailableModulesInfo { is_async: async_info.is_some(), }, )) @@ -126,21 +139,29 @@ pub async fn make_chunk_group( .into_iter() .collect(); let map = Vc::cell(map); - availability_info.with_chunk_items(map).await? + availability_info.with_modules(map).await? }; // Insert async chunk loaders for every referenced async module let async_loaders = async_modules .into_iter() - .map(|module| { + .map(async |module| { chunking_context .async_loader_chunk_item(*module, Value::new(availability_info)) .to_resolved() + .await }) .try_join() .await?; let has_async_loaders = !async_loaders.is_empty(); - let async_loader_chunk_items = async_loaders.iter().map(|&chunk_item| (chunk_item, None)); + let async_loader_chunk_items = + async_loaders + .iter() + .map(|&chunk_item| ChunkItemWithAsyncModuleInfo { + ty: ChunkItemTy::Included, + chunk_item, + async_info: None, + }); // And also add output assets referenced by async chunk loaders let async_loader_references = async_loaders @@ -155,36 +176,51 @@ pub async fn make_chunk_group( .collect(), ); - let mut referenced_output_assets = (*external_output_assets.await?).clone(); - referenced_output_assets.extend( - references_to_output_assets(&external_module_references) - .await? - .await? - .iter() - .copied(), - ); - - let rebased_modules = traced_modules + let traced_output_assets = traced_modules .into_iter() - .map(|module| { - RebasedAsset::new( - *module, - module.ident().path().root(), - module.ident().path().root(), - ) - .to_resolved() + .map(|module| async move { + Ok(ResolvedVc::upcast( + RebasedAsset::new( + *module, + module.ident().path().root(), + module.ident().path().root(), + ) + .to_resolved() + .await?, + )) }) .try_join() .await?; - referenced_output_assets.extend(rebased_modules.into_iter().map(ResolvedVc::upcast)); + let chunk_items = all_modules + .iter() + .map(|(m, async_info)| { + Either::Left(async move { + Ok(ChunkItemWithAsyncModuleInfo { + ty: ChunkItemTy::Included, + chunk_item: m.as_chunk_item(*chunking_context).to_resolved().await?, + async_info: *async_info, + }) + }) + }) + .chain(passthrough_modules.into_iter().map(|m| { + Either::Right(async move { + Ok(ChunkItemWithAsyncModuleInfo { + ty: ChunkItemTy::Passthrough, + chunk_item: m.as_chunk_item(*chunking_context).to_resolved().await?, + async_info: None, + }) + }) + })) + .try_join() + .await?; // Pass chunk items to chunking algorithm let mut chunks = make_chunks( *chunking_context, - Vc::cell(chunk_items.into_iter().collect()), + Vc::cell(chunk_items), "".into(), - Vc::cell(referenced_output_assets), + Vc::cell(traced_output_assets), ) .await? .clone_value(); diff --git a/turbopack/crates/turbopack-core/src/chunk/chunking.rs b/turbopack/crates/turbopack-core/src/chunk/chunking.rs index ee4ab1b13a7f9..0a9788b3bdc49 100644 --- a/turbopack/crates/turbopack-core/src/chunk/chunking.rs +++ b/turbopack/crates/turbopack-core/src/chunk/chunking.rs @@ -11,8 +11,8 @@ use turbo_rcstr::RcStr; use turbo_tasks::{FxIndexMap, ReadRef, ResolvedVc, TryJoinIterExt, ValueToString, Vc}; use super::{ - AsyncModuleInfo, Chunk, ChunkItem, ChunkItemsWithAsyncModuleInfo, ChunkType, ChunkingContext, - Chunks, + AsyncModuleInfo, Chunk, ChunkItem, ChunkItemTy, ChunkItemWithAsyncModuleInfo, + ChunkItemsWithAsyncModuleInfo, ChunkType, ChunkingContext, Chunks, }; use crate::output::OutputAssets; @@ -49,22 +49,26 @@ pub async fn make_chunks( key_prefix: RcStr, mut referenced_output_assets: Vc, ) -> Result> { + let chunk_items = chunk_items.await?; let chunk_items = chunk_items - .await? .iter() - .map(|&(chunk_item, async_info)| async move { - let chunk_item_info = - chunk_item_info(chunking_context, *chunk_item, async_info.map(|info| *info)) - .await?; - Ok((chunk_item, async_info, chunk_item_info)) + .map(|c| async move { + let chunk_item_info = chunk_item_info( + chunking_context, + *c.chunk_item, + c.async_info.map(|info| *info), + ) + .await?; + Ok((c, chunk_item_info)) }) .try_join() .await?; + let mut map = FxIndexMap::<_, Vec<_>>::default(); - for (chunk_item, async_info, chunk_item_info) in chunk_items { + for (c, chunk_item_info) in chunk_items { map.entry(chunk_item_info.ty) .or_default() - .push((chunk_item, async_info, chunk_item_info)); + .push((c, chunk_item_info)); } let mut chunks = Vec::new(); @@ -73,14 +77,24 @@ pub async fn make_chunks( let chunk_items = chunk_items .into_iter() - .map(|(chunk_item, async_info, chunk_item_info)| async move { - Ok(( - chunk_item, - async_info, - chunk_item_info.size, - chunk_item_info.name.await?, - )) - }) + .map( + async |( + ChunkItemWithAsyncModuleInfo { + ty, + chunk_item, + async_info, + }, + chunk_item_info, + )| { + Ok(ChunkItemWithInfo { + ty: *ty, + chunk_item: *chunk_item, + async_info: *async_info, + size: chunk_item_info.size, + asset_ident: chunk_item_info.name.await?, + }) + }, + ) .try_join() .await?; @@ -119,12 +133,13 @@ pub async fn make_chunks( Ok(Vc::cell(resolved_chunks)) } -type ChunkItemWithInfo = ( - ResolvedVc>, - Option>, - usize, - ReadRef, -); +struct ChunkItemWithInfo { + ty: ChunkItemTy, + chunk_item: ResolvedVc>, + async_info: Option>, + size: usize, + asset_ident: ReadRef, +} struct SplitContext<'a> { ty: ResolvedVc>, @@ -169,7 +184,18 @@ async fn make_chunk( split_context.chunking_context, chunk_items .into_iter() - .map(|(chunk_item, async_info, ..)| (chunk_item, async_info)) + .map( + |ChunkItemWithInfo { + ty, + chunk_item, + async_info, + .. + }| ChunkItemWithAsyncModuleInfo { + ty, + chunk_item, + async_info, + }, + ) .collect(), replace( split_context.referenced_output_assets, @@ -191,7 +217,7 @@ async fn app_vendors_split( let mut app_chunk_items = Vec::new(); let mut vendors_chunk_items = Vec::new(); for item in chunk_items { - let (_, _, _, asset_ident) = &item; + let ChunkItemWithInfo { asset_ident, .. } = &item; if is_app_code(asset_ident) { app_chunk_items.push(item); } else { @@ -239,7 +265,7 @@ async fn package_name_split( ) -> Result<()> { let mut map = FxIndexMap::<_, Vec>::default(); for item in chunk_items { - let (_, _, _, asset_ident) = &item; + let ChunkItemWithInfo { asset_ident, .. } = &item; let package_name = package_name(asset_ident); if let Some(list) = map.get_mut(package_name) { list.push(item); @@ -273,7 +299,7 @@ async fn folder_split( let mut map = FxIndexMap::<_, (_, Vec)>::default(); loop { for item in chunk_items { - let (_, _, _, asset_ident) = &item; + let ChunkItemWithInfo { asset_ident, .. } = &item; let (folder_name, new_location) = folder_name(asset_ident, location); if let Some((_, list)) = map.get_mut(folder_name) { list.push(item); @@ -316,7 +342,7 @@ async fn folder_split( } } if !remaining.is_empty() { - let (_, _, _, asset_ident) = &remaining[0]; + let ChunkItemWithInfo { asset_ident, .. } = &remaining[0]; let mut key = format!("{}-{}", name, &asset_ident[..location]); if !handle_split_group(&mut remaining, &mut key, split_context, None).await? { make_chunk(remaining, &mut key, split_context).await?; @@ -365,7 +391,7 @@ enum ChunkSize { /// large or perfect fit. fn chunk_size(chunk_items: &[ChunkItemWithInfo]) -> ChunkSize { let mut total_size = 0; - for (_, _, size, _) in chunk_items { + for ChunkItemWithInfo { size, .. } in chunk_items { total_size += size; } if total_size >= LARGE_CHUNK { diff --git a/turbopack/crates/turbopack-core/src/chunk/mod.rs b/turbopack/crates/turbopack-core/src/chunk/mod.rs index fbf9e42d0cf13..3523ea619ee0f 100644 --- a/turbopack/crates/turbopack-core/src/chunk/mod.rs +++ b/turbopack/crates/turbopack-core/src/chunk/mod.rs @@ -1,5 +1,5 @@ pub mod availability_info; -pub mod available_chunk_items; +pub mod available_modules; pub mod chunk_group; pub mod chunking; pub(crate) mod chunking_context; @@ -31,7 +31,7 @@ use turbo_tasks::{ use turbo_tasks_fs::FileSystemPath; use turbo_tasks_hash::DeterministicHash; -use self::{availability_info::AvailabilityInfo, available_chunk_items::AvailableChunkItems}; +use self::{availability_info::AvailabilityInfo, available_modules::AvailableModules}; pub use self::{ chunking_context::{ ChunkGroupResult, ChunkGroupType, ChunkingContext, ChunkingContextExt, @@ -41,11 +41,7 @@ pub use self::{ evaluate::{EvaluatableAsset, EvaluatableAssetExt, EvaluatableAssets}, }; use crate::{ - asset::Asset, - environment::ChunkLoading, - ident::AssetIdent, - module::Module, - output::{OutputAsset, OutputAssets}, + asset::Asset, ident::AssetIdent, module::Module, output::OutputAssets, reference::ModuleReference, }; @@ -209,14 +205,14 @@ pub trait ChunkableModuleReference: ModuleReference + ValueToString { } } -type AsyncInfo = FxIndexMap>, Vec>>>; +type AsyncInfo = + FxIndexMap>, Vec>>>; pub struct ChunkContentResult { - pub chunk_items: FxIndexSet>>, + pub chunkable_modules: FxIndexSet>>, pub async_modules: FxIndexSet>>, pub traced_modules: FxIndexSet>>, - pub external_output_assets: ResolvedVc, - pub external_module_references: FxIndexSet>>, + pub passthrough_modules: FxIndexSet>>, /// A map from local module to all children from which the async module /// status is inherited pub forward_edges_inherit_async: AsyncInfo, @@ -229,11 +225,18 @@ pub struct ChunkContentResult { } pub async fn chunk_content( - chunking_context: ResolvedVc>, chunk_entries: impl IntoIterator>>, availability_info: AvailabilityInfo, + can_split_async: bool, + should_trace: bool, ) -> Result { - chunk_content_internal_parallel(chunking_context, chunk_entries, availability_info).await + chunk_content_internal_parallel( + chunk_entries, + availability_info, + can_split_async, + should_trace, + ) + .await } #[derive( @@ -252,14 +255,14 @@ enum InheritAsyncEdge { #[derive(Eq, PartialEq, Clone, Hash, Serialize, Deserialize, TraceRawVcs, Debug, NonLocalValue)] enum ChunkContentGraphNode { - // A chunk item not placed in the current chunk, but whose references we will + // A module not placed in the current chunk, but whose references we will // follow to find more graph nodes. - PassthroughChunkItem { - item: ResolvedVc>, + PassthroughModule { + module: ResolvedVc>, }, - // Chunk items that are placed into the current chunk group - ChunkItem { - item: ResolvedVc>, + // Modules that are placed into the current chunk group + Module { + module: ResolvedVc>, ident: ReadRef, }, // Async module that is referenced from the chunk group @@ -270,21 +273,18 @@ enum ChunkContentGraphNode { TracedModule { module: ResolvedVc>, }, - ExternalOutputAssets(ResolvedVc), - // ModuleReferences that are not placed in the current chunk group - ExternalModuleReference(ResolvedVc>), - /// A list of directly referenced chunk items from which `is_async_module` + /// A list of directly referenced modules from which `is_async_module` /// will be inherited. InheritAsyncInfo { - item: ResolvedVc>, - references: Vec<(ResolvedVc>, InheritAsyncEdge)>, + module: ResolvedVc>, + references: Vec<(ResolvedVc>, InheritAsyncEdge)>, }, } #[derive(Debug, Clone, Copy, TaskInput, PartialEq, Eq, Hash, Serialize, Deserialize)] enum ChunkGraphNodeToReferences { - PassthroughChunkItem(ResolvedVc>), - ChunkItem(ResolvedVc>), + PassthroughModule(ResolvedVc>), + Module(ResolvedVc>), } #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, TraceRawVcs, NonLocalValue)] @@ -298,37 +298,37 @@ struct ChunkGraphEdge { struct ChunkGraphEdges(Vec); #[turbo_tasks::function] -async fn graph_node_to_referenced_nodes_with_available_chunk_items( +async fn graph_node_to_referenced_nodes_with_available_modules( node: ChunkGraphNodeToReferences, - chunking_context: Vc>, - available_chunk_items: Vc, + available_modules: Vc, + can_split_async: bool, + should_trace: bool, ) -> Result> { - let edges = graph_node_to_referenced_nodes(node, chunking_context); + let edges = graph_node_to_referenced_nodes(node, can_split_async, should_trace); let edges_ref = edges.await?; for (unchanged, edge) in edges_ref.iter().enumerate() { - if let ChunkContentGraphNode::ChunkItem { item, .. } = edge.node { - let available_chunk_items = available_chunk_items.await?; - if let Some(info) = available_chunk_items.get(item).await? { + if let ChunkContentGraphNode::Module { module, .. } = edge.node { + if let Some(info) = *available_modules.get(*module).await? { let mut new_edges = Vec::with_capacity(edges_ref.len()); new_edges.extend(edges_ref[0..unchanged].iter().cloned()); - let mut available_chunk_item_info = HashMap::new(); - available_chunk_item_info.insert(item, info); + let mut available_module_info = HashMap::new(); + available_module_info.insert(module, info); for edge in edges_ref[unchanged + 1..].iter() { match edge.node { - ChunkContentGraphNode::ChunkItem { item, .. } => { - if let Some(info) = available_chunk_items.get(item).await? { - available_chunk_item_info.insert(item, info); + ChunkContentGraphNode::Module { module, .. } => { + if let Some(info) = *available_modules.get(*module).await? { + available_module_info.insert(module, info); continue; } } ChunkContentGraphNode::InheritAsyncInfo { - item, + module, ref references, } => { let new_references = references .iter() .filter_map(|&(r, _)| { - if let Some(info) = available_chunk_item_info.get(&r) { + if let Some(info) = available_module_info.get(&r) { if info.is_async { Some((r, InheritAsyncEdge::AvailableAsyncModule)) } else { @@ -342,7 +342,7 @@ async fn graph_node_to_referenced_nodes_with_available_chunk_items( new_edges.push(ChunkGraphEdge { key: edge.key, node: ChunkContentGraphNode::InheritAsyncInfo { - item, + module, references: new_references, }, }); @@ -362,36 +362,27 @@ async fn graph_node_to_referenced_nodes_with_available_chunk_items( #[turbo_tasks::function] async fn graph_node_to_referenced_nodes( node: ChunkGraphNodeToReferences, - chunking_context: Vc>, + can_split_async: bool, + should_trace: bool, ) -> Result> { - let (parent, module_references, output_asset_references) = match &node { - ChunkGraphNodeToReferences::PassthroughChunkItem(item) => { - (None, item.module().references(), item.references()) - } - ChunkGraphNodeToReferences::ChunkItem(item) => { - (Some(*item), item.module().references(), item.references()) - } + let (parent, module_references) = match &node { + ChunkGraphNodeToReferences::PassthroughModule(item) => (None, item.references()), + ChunkGraphNodeToReferences::Module(item) => (Some(*item), item.references()), }; let module_references = module_references.await?; - let mut graph_nodes = module_references + let graph_nodes = module_references .iter() .map(|reference| async { let reference = *reference; let Some(chunkable_module_reference) = ResolvedVc::try_downcast::>(reference).await? else { - return Ok(vec![ChunkGraphEdge { - key: None, - node: ChunkContentGraphNode::ExternalModuleReference(reference), - }]); + return Ok(vec![]); }; let Some(chunking_type) = &*chunkable_module_reference.chunking_type().await? else { - return Ok(vec![ChunkGraphEdge { - key: None, - node: ChunkContentGraphNode::ExternalModuleReference(reference), - }]); + return Ok(vec![]); }; let module_data = reference @@ -403,7 +394,7 @@ async fn graph_node_to_referenced_nodes( .into_iter() .map(|&module| async move { if matches!(chunking_type, ChunkingType::Traced) { - if *chunking_context.is_tracing_enabled().await? { + if should_trace { return Ok(( Some(ChunkGraphEdge { key: None, @@ -419,78 +410,46 @@ async fn graph_node_to_referenced_nodes( let Some(chunkable_module) = ResolvedVc::try_sidecast::>(module).await? else { - return Ok(( + return Ok((None, None)); + }; + + match chunking_type { + ChunkingType::Parallel => Ok(( + Some(ChunkGraphEdge { + key: Some(module), + node: ChunkContentGraphNode::Module { + module: chunkable_module, + ident: module.ident().to_string().await?, + }, + }), + None, + )), + ChunkingType::ParallelInheritAsync => Ok(( + Some(ChunkGraphEdge { + key: Some(module), + node: ChunkContentGraphNode::Module { + module: chunkable_module, + ident: module.ident().to_string().await?, + }, + }), + Some((chunkable_module, InheritAsyncEdge::LocalModule)), + )), + ChunkingType::Passthrough => Ok(( Some(ChunkGraphEdge { key: None, - node: ChunkContentGraphNode::ExternalModuleReference(reference), + node: ChunkContentGraphNode::PassthroughModule { + module: chunkable_module, + }, }), None, - )); - }; - - match chunking_type { - ChunkingType::Parallel => { - let chunk_item = chunkable_module - .as_chunk_item(chunking_context) - .to_resolved() - .await?; - Ok(( - Some(ChunkGraphEdge { - key: Some(module), - node: ChunkContentGraphNode::ChunkItem { - item: chunk_item, - ident: module.ident().to_string().await?, - }, - }), - None, - )) - } - ChunkingType::ParallelInheritAsync => { - let chunk_item = chunkable_module - .as_chunk_item(chunking_context) - .to_resolved() - .await?; - Ok(( - Some(ChunkGraphEdge { - key: Some(module), - node: ChunkContentGraphNode::ChunkItem { - item: chunk_item, - ident: module.ident().to_string().await?, - }, - }), - Some((chunk_item, InheritAsyncEdge::LocalModule)), - )) - } - ChunkingType::Passthrough => { - let chunk_item = chunkable_module - .as_chunk_item(chunking_context) - .to_resolved() - .await?; - - Ok(( - Some(ChunkGraphEdge { - key: None, - node: ChunkContentGraphNode::PassthroughChunkItem { - item: chunk_item, - }, - }), - None, - )) - } + )), ChunkingType::Async => { - let chunk_loading = - chunking_context.environment().chunk_loading().await?; - if matches!(*chunk_loading, ChunkLoading::Edge) { - let chunk_item = chunkable_module - .as_chunk_item(chunking_context) - .to_resolved() - .await?; + if can_split_async { Ok(( Some(ChunkGraphEdge { - key: Some(module), - node: ChunkContentGraphNode::ChunkItem { - item: chunk_item, - ident: module.ident().to_string().await?, + key: None, + node: ChunkContentGraphNode::AsyncModule { + module: chunkable_module, }, }), None, @@ -498,9 +457,10 @@ async fn graph_node_to_referenced_nodes( } else { Ok(( Some(ChunkGraphEdge { - key: None, - node: ChunkContentGraphNode::AsyncModule { + key: Some(module), + node: ChunkContentGraphNode::Module { module: chunkable_module, + ident: module.ident().to_string().await?, }, }), None, @@ -535,7 +495,7 @@ async fn graph_node_to_referenced_nodes( graph_nodes.push(ChunkGraphEdge { key: None, node: ChunkContentGraphNode::InheritAsyncInfo { - item: parent, + module: parent, references: inherit_async_references, }, }) @@ -547,20 +507,14 @@ async fn graph_node_to_referenced_nodes( .try_flat_join() .await?; - graph_nodes.push(ChunkGraphEdge { - key: None, - node: ChunkContentGraphNode::ExternalOutputAssets( - output_asset_references.to_resolved().await?, - ), - }); - Ok(Vc::cell(graph_nodes)) } struct ChunkContentVisit { - chunking_context: ResolvedVc>, - available_chunk_items: Option>, + available_chunk_items: Option>, processed_modules: HashSet>>, + should_trace: bool, + can_split_async: bool, } type ChunkItemToGraphNodesEdges = impl Iterator; @@ -575,7 +529,7 @@ impl Visit for ChunkContentVisit { fn visit(&mut self, edge: ChunkGraphEdge) -> VisitControlFlow { let ChunkGraphEdge { key, node } = edge; let Some(module) = key else { - if matches!(node, ChunkContentGraphNode::PassthroughChunkItem { .. }) { + if matches!(node, ChunkContentGraphNode::PassthroughModule { .. }) { return VisitControlFlow::Continue(node); } else { // All other types don't have edges @@ -593,16 +547,17 @@ impl Visit for ChunkContentVisit { fn edges(&mut self, node: &ChunkContentGraphNode) -> Self::EdgesFuture { let node = node.clone(); - let chunking_context = self.chunking_context; let available_chunk_items = self.available_chunk_items; + let can_split_async = self.can_split_async; + let should_trace = self.should_trace; async move { let node = match node { - ChunkContentGraphNode::PassthroughChunkItem { item } => { - ChunkGraphNodeToReferences::PassthroughChunkItem(item) + ChunkContentGraphNode::PassthroughModule { module: item } => { + ChunkGraphNodeToReferences::PassthroughModule(item) } - ChunkContentGraphNode::ChunkItem { item, .. } => { - ChunkGraphNodeToReferences::ChunkItem(item) + ChunkContentGraphNode::Module { module: item, .. } => { + ChunkGraphNodeToReferences::Module(item) } _ => { return Ok(None.into_iter().flatten()); @@ -610,13 +565,14 @@ impl Visit for ChunkContentVisit { }; let nodes = if let Some(available_chunk_items) = available_chunk_items { - graph_node_to_referenced_nodes_with_available_chunk_items( + graph_node_to_referenced_nodes_with_available_modules( node, - *chunking_context, *available_chunk_items, + can_split_async, + should_trace, ) } else { - graph_node_to_referenced_nodes(node, *chunking_context) + graph_node_to_referenced_nodes(node, can_split_async, should_trace) } .await?; Ok(Some(nodes.into_iter().cloned()).into_iter().flatten()) @@ -624,7 +580,7 @@ impl Visit for ChunkContentVisit { } fn span(&mut self, node: &ChunkContentGraphNode) -> Span { - if let ChunkContentGraphNode::ChunkItem { ident, .. } = node { + if let ChunkContentGraphNode::Module { ident, .. } = node { info_span!("chunking module", name = display(ident)) } else { Span::current() @@ -633,9 +589,10 @@ impl Visit for ChunkContentVisit { } async fn chunk_content_internal_parallel( - chunking_context: ResolvedVc>, chunk_entries: impl IntoIterator>>, availability_info: AvailabilityInfo, + can_split_async: bool, + should_trace: bool, ) -> Result { let root_edges = chunk_entries .into_iter() @@ -647,11 +604,8 @@ async fn chunk_content_internal_parallel( }; Ok(Some(ChunkGraphEdge { key: Some(entry), - node: ChunkContentGraphNode::ChunkItem { - item: chunkable_module - .as_chunk_item(*chunking_context) - .to_resolved() - .await?, + node: ChunkContentGraphNode::Module { + module: chunkable_module, ident: chunkable_module.ident().to_string().await?, }, })) @@ -660,9 +614,10 @@ async fn chunk_content_internal_parallel( .await?; let visit = ChunkContentVisit { - chunking_context, - available_chunk_items: availability_info.available_chunk_items(), + available_chunk_items: availability_info.available_modules(), processed_modules: Default::default(), + can_split_async, + should_trace, }; let GraphTraversalResult::Completed(traversal_result) = @@ -673,11 +628,9 @@ async fn chunk_content_internal_parallel( let graph_nodes: Vec<_> = traversal_result?.into_reverse_topological().collect(); - let mut chunk_items = FxIndexSet::default(); + let mut chunkable_modules = FxIndexSet::default(); let mut async_modules = FxIndexSet::default(); - let mut external_module_references = FxIndexSet::default(); - let mut external_output_assets: FxIndexSet>> = - FxIndexSet::default(); + let mut passthrough_modules = FxIndexSet::default(); let mut forward_edges_inherit_async = FxIndexMap::default(); let mut local_back_edges_inherit_async = FxIndexMap::default(); let mut available_async_modules_back_edges_inherit_async = FxIndexMap::default(); @@ -685,25 +638,22 @@ async fn chunk_content_internal_parallel( for graph_node in graph_nodes { match graph_node { - ChunkContentGraphNode::PassthroughChunkItem { .. } => {} + ChunkContentGraphNode::PassthroughModule { module } => { + passthrough_modules.insert(module); + } ChunkContentGraphNode::TracedModule { module } => { traced_modules.insert(module); } - ChunkContentGraphNode::ChunkItem { item, .. } => { - chunk_items.insert(item); + ChunkContentGraphNode::Module { module: item, .. } => { + chunkable_modules.insert(item); } ChunkContentGraphNode::AsyncModule { module } => { async_modules.insert(module); } - ChunkContentGraphNode::ExternalModuleReference(reference) => { - external_module_references.insert(reference); - } - ChunkContentGraphNode::ExternalOutputAssets(reference) => { - for output_asset in reference.await? { - external_output_assets.insert(*output_asset); - } - } - ChunkContentGraphNode::InheritAsyncInfo { item, references } => { + ChunkContentGraphNode::InheritAsyncInfo { + module: item, + references, + } => { for &(reference, ty) in &references { match ty { InheritAsyncEdge::LocalModule => local_back_edges_inherit_async @@ -727,11 +677,10 @@ async fn chunk_content_internal_parallel( } Ok(ChunkContentResult { - chunk_items, + chunkable_modules, async_modules, traced_modules, - external_output_assets: ResolvedVc::cell(external_output_assets.into_iter().collect()), - external_module_references, + passthrough_modules, forward_edges_inherit_async, local_back_edges_inherit_async, available_async_modules_back_edges_inherit_async, @@ -764,10 +713,6 @@ pub trait ChunkItem { fn module(self: Vc) -> Vc>; fn chunking_context(self: Vc) -> Vc>; - - fn is_self_async(self: Vc) -> Vc { - Vc::cell(false) - } } #[turbo_tasks::value_trait] @@ -801,14 +746,14 @@ pub struct ChunkItems(pub Vec>>); #[turbo_tasks::value] pub struct AsyncModuleInfo { - pub referenced_async_modules: AutoSet>>, + pub referenced_async_modules: AutoSet>>, } #[turbo_tasks::value_impl] impl AsyncModuleInfo { #[turbo_tasks::function] pub async fn new( - referenced_async_modules: Vec>>, + referenced_async_modules: Vec>>, ) -> Result> { Ok(Self { referenced_async_modules: referenced_async_modules.into_iter().collect(), @@ -817,10 +762,35 @@ impl AsyncModuleInfo { } } -pub type ChunkItemWithAsyncModuleInfo = ( - ResolvedVc>, - Option>, -); +#[derive( + Copy, + Debug, + Clone, + Serialize, + Deserialize, + PartialEq, + Eq, + Hash, + TraceRawVcs, + TaskInput, + NonLocalValue, +)] +pub enum ChunkItemTy { + /// The ChunkItem should be included as content in the chunk. + Included, + /// The ChunkItem should be used to trace references but should not included in the chunk. + Passthrough, +} + +#[derive( + Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash, TraceRawVcs, TaskInput, NonLocalValue, +)] +// #[turbo_tasks::value] +pub struct ChunkItemWithAsyncModuleInfo { + pub ty: ChunkItemTy, + pub chunk_item: ResolvedVc>, + pub async_info: Option>, +} #[turbo_tasks::value(transparent)] pub struct ChunkItemsWithAsyncModuleInfo(Vec); diff --git a/turbopack/crates/turbopack-core/src/module.rs b/turbopack/crates/turbopack-core/src/module.rs index f1d9f51bcfec0..1b1429a7389ef 100644 --- a/turbopack/crates/turbopack-core/src/module.rs +++ b/turbopack/crates/turbopack-core/src/module.rs @@ -19,6 +19,11 @@ pub trait Module: Asset { fn additional_layers_modules(self: Vc) -> Vc { Vc::cell(vec![]) } + + /// Signifies the module itself is async, e.g. it uses top-level await, is a wasm module, etc. + fn is_self_async(self: Vc) -> Vc { + Vc::cell(false) + } } #[turbo_tasks::value(transparent)] diff --git a/turbopack/crates/turbopack-css/src/chunk/mod.rs b/turbopack/crates/turbopack-css/src/chunk/mod.rs index 0601eccd78b1a..05fddde51265e 100644 --- a/turbopack/crates/turbopack-css/src/chunk/mod.rs +++ b/turbopack/crates/turbopack-css/src/chunk/mod.rs @@ -3,7 +3,7 @@ pub mod source_map; use std::fmt::Write; -use anyhow::{bail, Context, Result}; +use anyhow::{bail, Result}; use turbo_rcstr::RcStr; use turbo_tasks::{FxIndexSet, ResolvedVc, TryJoinIterExt, Value, ValueDefault, ValueToString, Vc}; use turbo_tasks_fs::{rope::Rope, File, FileSystem}; @@ -503,11 +503,14 @@ impl ChunkType for CssChunkType { let content = CssChunkContent { chunk_items: chunk_items .iter() - .map(|(chunk_item, _async_info)| async move { + .map(async |ChunkItemWithAsyncModuleInfo { chunk_item, .. }| { + let Some(chunk_item) = + ResolvedVc::try_downcast::>(*chunk_item).await? + else { + bail!("Chunk item is not an css chunk item but reporting chunk type css"); + }; // CSS doesn't need to care about async_info, so we can discard it - ResolvedVc::try_downcast::>(*chunk_item) - .await? - .context("Chunk item is not an css chunk item but reporting chunk type css") + Ok(chunk_item) }) .try_join() .await?, diff --git a/turbopack/crates/turbopack-css/src/lib.rs b/turbopack/crates/turbopack-css/src/lib.rs index c4773f56c1817..f4b3ac450531c 100644 --- a/turbopack/crates/turbopack-css/src/lib.rs +++ b/turbopack/crates/turbopack-css/src/lib.rs @@ -1,3 +1,4 @@ +#![feature(async_closure)] #![feature(min_specialization)] #![feature(box_patterns)] #![feature(iter_intersperse)] diff --git a/turbopack/crates/turbopack-ecmascript/src/async_chunk/chunk_item.rs b/turbopack/crates/turbopack-ecmascript/src/async_chunk/chunk_item.rs index c15fc5b8ea018..2933fa1b5aac0 100644 --- a/turbopack/crates/turbopack-ecmascript/src/async_chunk/chunk_item.rs +++ b/turbopack/crates/turbopack-ecmascript/src/async_chunk/chunk_item.rs @@ -32,19 +32,8 @@ impl AsyncLoaderChunkItem { #[turbo_tasks::function] pub(super) async fn chunks(&self) -> Result> { let module = self.module.await?; - if let Some(chunk_items) = module.availability_info.available_chunk_items() { - if chunk_items - .await? - .get( - module - .inner - .as_chunk_item(*ResolvedVc::upcast(self.chunking_context)) - .to_resolved() - .await?, - ) - .await? - .is_some() - { + if let Some(chunk_items) = module.availability_info.available_modules() { + if chunk_items.get(*module.inner).await?.is_some() { return Ok(Vc::cell(vec![])); } } @@ -168,7 +157,7 @@ impl ChunkItem for AsyncLoaderChunkItem { async fn content_ident(&self) -> Result> { let mut ident = self.module.ident(); if let Some(available_chunk_items) = - self.module.await?.availability_info.available_chunk_items() + self.module.await?.availability_info.available_modules() { ident = ident.with_modifier(Vc::cell( available_chunk_items.hash().await?.to_string().into(), diff --git a/turbopack/crates/turbopack-ecmascript/src/chunk/chunk_type.rs b/turbopack/crates/turbopack-ecmascript/src/chunk/chunk_type.rs index 52d4aba0dd2cb..caf4cb8a360e4 100644 --- a/turbopack/crates/turbopack-ecmascript/src/chunk/chunk_type.rs +++ b/turbopack/crates/turbopack-ecmascript/src/chunk/chunk_type.rs @@ -9,7 +9,10 @@ use turbopack_core::{ output::OutputAssets, }; -use super::{EcmascriptChunk, EcmascriptChunkContent, EcmascriptChunkItem}; +use super::{ + item::EcmascriptChunkItemWithAsyncInfo, EcmascriptChunk, EcmascriptChunkContent, + EcmascriptChunkItem, +}; #[turbo_tasks::value] #[derive(Default)] @@ -45,18 +48,28 @@ impl ChunkType for EcmascriptChunkType { let content = EcmascriptChunkContent { chunk_items: chunk_items .iter() - .map(|&(chunk_item, async_info)| async move { - let Some(chunk_item) = - ResolvedVc::try_downcast::>(chunk_item) - .await? - else { - bail!( - "Chunk item is not an ecmascript chunk item but reporting chunk type \ - ecmascript" - ); - }; - Ok((chunk_item, async_info)) - }) + .map( + async |ChunkItemWithAsyncModuleInfo { + ty, + chunk_item, + async_info, + }| { + let Some(chunk_item) = + ResolvedVc::try_downcast::>(*chunk_item) + .await? + else { + bail!( + "Chunk item is not an ecmascript chunk item but reporting chunk \ + type ecmascript" + ); + }; + Ok(EcmascriptChunkItemWithAsyncInfo { + ty: *ty, + chunk_item, + async_info: *async_info, + }) + }, + ) .try_join() .await?, referenced_output_assets: referenced_output_assets.await?.clone_value(), diff --git a/turbopack/crates/turbopack-ecmascript/src/chunk/content.rs b/turbopack/crates/turbopack-ecmascript/src/chunk/content.rs index bff643851d383..8939611d2145f 100644 --- a/turbopack/crates/turbopack-ecmascript/src/chunk/content.rs +++ b/turbopack/crates/turbopack-ecmascript/src/chunk/content.rs @@ -1,15 +1,35 @@ -use turbo_tasks::ResolvedVc; -use turbopack_core::{chunk::AsyncModuleInfo, output::OutputAsset}; +use anyhow::Result; +use turbo_tasks::{ResolvedVc, Vc}; +use turbopack_core::{ + chunk::{ChunkItem, ChunkItemTy, ChunkItems}, + output::OutputAsset, +}; -use super::item::EcmascriptChunkItem; - -type EcmascriptChunkItemWithAsyncInfo = ( - ResolvedVc>, - Option>, -); +use super::item::EcmascriptChunkItemWithAsyncInfo; #[turbo_tasks::value(shared)] pub struct EcmascriptChunkContent { pub chunk_items: Vec, pub referenced_output_assets: Vec>>, } + +#[turbo_tasks::value_impl] +impl EcmascriptChunkContent { + #[turbo_tasks::function] + pub async fn included_chunk_items(&self) -> Result> { + Ok(ChunkItems( + self.chunk_items + .iter() + .filter_map(|EcmascriptChunkItemWithAsyncInfo { ty, chunk_item, .. }| { + if matches!(ty, ChunkItemTy::Included) { + Some(chunk_item) + } else { + None + } + }) + .map(|item| ResolvedVc::upcast::>(*item)) + .collect(), + ) + .cell()) + } +} diff --git a/turbopack/crates/turbopack-ecmascript/src/chunk/item.rs b/turbopack/crates/turbopack-ecmascript/src/chunk/item.rs index 884575eb32585..4934b81a562aa 100644 --- a/turbopack/crates/turbopack-ecmascript/src/chunk/item.rs +++ b/turbopack/crates/turbopack-ecmascript/src/chunk/item.rs @@ -2,10 +2,12 @@ use std::io::Write; use anyhow::{bail, Result}; use serde::{Deserialize, Serialize}; -use turbo_tasks::{trace::TraceRawVcs, NonLocalValue, ResolvedVc, Upcast, ValueToString, Vc}; +use turbo_tasks::{ + trace::TraceRawVcs, NonLocalValue, ResolvedVc, TaskInput, Upcast, ValueToString, Vc, +}; use turbo_tasks_fs::{rope::Rope, FileSystemPath}; use turbopack_core::{ - chunk::{AsyncModuleInfo, ChunkItem, ChunkItemExt, ChunkingContext}, + chunk::{AsyncModuleInfo, ChunkItem, ChunkItemExt, ChunkItemTy, ChunkingContext}, code_builder::{fileify_source_map, Code, CodeBuilder}, error::PrettyPrintError, issue::{code_gen::CodeGenerationIssue, IssueExt, IssueSeverity, StyledString}, @@ -211,6 +213,15 @@ pub struct EcmascriptChunkItemOptions { pub placeholder_for_future_extensions: (), } +#[derive( + Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash, TraceRawVcs, TaskInput, NonLocalValue, +)] +pub struct EcmascriptChunkItemWithAsyncInfo { + pub ty: ChunkItemTy, + pub chunk_item: ResolvedVc>, + pub async_info: Option>, +} + #[turbo_tasks::value_trait] pub trait EcmascriptChunkItem: ChunkItem { fn content(self: Vc) -> Vc; diff --git a/turbopack/crates/turbopack-ecmascript/src/chunk/mod.rs b/turbopack/crates/turbopack-ecmascript/src/chunk/mod.rs index acbef76d16dea..537977d7e714a 100644 --- a/turbopack/crates/turbopack-ecmascript/src/chunk/mod.rs +++ b/turbopack/crates/turbopack-ecmascript/src/chunk/mod.rs @@ -8,7 +8,7 @@ use std::fmt::Write; use anyhow::{bail, Result}; use turbo_rcstr::RcStr; -use turbo_tasks::{ResolvedVc, TryJoinIterExt, Value, ValueToString, Vc}; +use turbo_tasks::{ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Value, ValueToString, Vc}; use turbo_tasks_fs::FileSystem; use turbopack_core::{ asset::{Asset, AssetContent}, @@ -19,7 +19,7 @@ use turbopack_core::{ utils::{children_from_output_assets, content_to_details}, Introspectable, IntrospectableChildren, }, - output::OutputAssets, + output::{OutputAsset, OutputAssets}, server_fs::ServerFileSystem, }; @@ -29,7 +29,7 @@ pub use self::{ data::EcmascriptChunkData, item::{ EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkItemExt, - EcmascriptChunkItemOptions, + EcmascriptChunkItemOptions, EcmascriptChunkItemWithAsyncInfo, }, placeable::{EcmascriptChunkPlaceable, EcmascriptExports}, }; @@ -75,8 +75,8 @@ fn availability_root_key() -> Vc { impl Chunk for EcmascriptChunk { #[turbo_tasks::function] async fn ident(&self) -> Result> { - let EcmascriptChunkContent { chunk_items, .. } = &*self.content.await?; - let mut common_path = if let Some((chunk_item, _)) = chunk_items.first() { + let chunk_items = &*self.content.included_chunk_items().await?; + let mut common_path = if let Some(chunk_item) = chunk_items.first() { let path = chunk_item.asset_ident().path().to_resolved().await?; Some((path, path.await?)) } else { @@ -84,7 +84,7 @@ impl Chunk for EcmascriptChunk { }; // The included chunk items describe the chunk uniquely - for &(chunk_item, _) in chunk_items.iter() { + for &chunk_item in chunk_items.iter() { if let Some((common_path_vc, common_path_ref)) = common_path.as_mut() { let path = chunk_item.asset_ident().path().await?; while !path.is_inside_or_equal_ref(common_path_ref) { @@ -102,7 +102,7 @@ impl Chunk for EcmascriptChunk { let chunk_item_key = chunk_item_key().to_resolved().await?; let assets = chunk_items .iter() - .map(|&(chunk_item, _)| async move { + .map(|&chunk_item| async move { Ok(( chunk_item_key, chunk_item.content_ident().to_resolved().await?, @@ -136,19 +136,26 @@ impl Chunk for EcmascriptChunk { #[turbo_tasks::function] async fn references(&self) -> Result> { let content = self.content.await?; - Ok(Vc::cell(content.referenced_output_assets.clone())) + let mut referenced_output_assets: Vec>> = content + .chunk_items + .iter() + .map(async |with_info| { + Ok(with_info + .chunk_item + .references() + .await? + .into_iter() + .copied()) + }) + .try_flat_join() + .await?; + referenced_output_assets.extend(content.referenced_output_assets.iter().copied()); + Ok(Vc::cell(referenced_output_assets)) } #[turbo_tasks::function] - async fn chunk_items(&self) -> Result> { - let EcmascriptChunkContent { chunk_items, .. } = &*self.content.await?; - Ok(ChunkItems( - chunk_items - .iter() - .map(|&(item, _)| ResolvedVc::upcast(item)) - .collect(), - ) - .cell()) + async fn chunk_items(&self) -> Vc { + self.content.included_chunk_items() } } @@ -210,9 +217,8 @@ impl Introspectable for EcmascriptChunk { let content = content_to_details(self.content()); let mut details = String::new(); let this = self.await?; - let chunk_content = this.content.await?; details += "Chunk items:\n\n"; - for (chunk_item, _) in chunk_content.chunk_items.iter() { + for chunk_item in this.content.included_chunk_items().await? { writeln!(details, "- {}", chunk_item.asset_ident().to_string().await?)?; } details += "\nContent:\n\n"; @@ -226,7 +232,7 @@ impl Introspectable for EcmascriptChunk { .await? .clone_value(); let chunk_item_module_key = chunk_item_module_key().to_resolved().await?; - for &(chunk_item, _) in self.await?.content.await?.chunk_items.iter() { + for chunk_item in self.await?.content.included_chunk_items().await? { children.insert(( chunk_item_module_key, IntrospectableModule::new(chunk_item.module()) diff --git a/turbopack/crates/turbopack-ecmascript/src/lib.rs b/turbopack/crates/turbopack-ecmascript/src/lib.rs index a2ec944b7befd..a5892d68b146d 100644 --- a/turbopack/crates/turbopack-ecmascript/src/lib.rs +++ b/turbopack/crates/turbopack-ecmascript/src/lib.rs @@ -1,5 +1,6 @@ // Needed for swc visit_ macros #![allow(non_local_definitions)] +#![feature(async_closure)] #![feature(box_patterns)] #![feature(min_specialization)] #![feature(iter_intersperse)] @@ -548,6 +549,15 @@ impl Module for EcmascriptModuleAsset { let references = analyze.references.await?.iter().copied().collect(); Ok(Vc::cell(references)) } + + #[turbo_tasks::function] + async fn is_self_async(self: Vc) -> Result> { + if let Some(async_module) = *self.get_async_module().await? { + Ok(async_module.is_self_async(*self.analyze().await?.references)) + } else { + Ok(Vc::cell(false)) + } + } } #[turbo_tasks::value_impl] @@ -643,15 +653,6 @@ impl ChunkItem for ModuleChunkItem { fn module(&self) -> Vc> { *ResolvedVc::upcast(self.module) } - - #[turbo_tasks::function] - async fn is_self_async(&self) -> Result> { - if let Some(async_module) = *self.module.get_async_module().await? { - Ok(async_module.is_self_async(*self.module.analyze().await?.references)) - } else { - Ok(Vc::cell(false)) - } - } } #[turbo_tasks::value_impl] @@ -735,11 +736,7 @@ impl EcmascriptModuleContent { } } if let Some(async_module) = *async_module.await? { - code_gens.push(async_module.code_generation( - chunking_context, - async_module_info, - references, - )); + code_gens.push(async_module.code_generation(async_module_info, references)); } for c in code_generation.await?.iter() { match c { diff --git a/turbopack/crates/turbopack-ecmascript/src/manifest/chunk_asset.rs b/turbopack/crates/turbopack-ecmascript/src/manifest/chunk_asset.rs index 1ca6696f1d10e..d428825f01189 100644 --- a/turbopack/crates/turbopack-ecmascript/src/manifest/chunk_asset.rs +++ b/turbopack/crates/turbopack-ecmascript/src/manifest/chunk_asset.rs @@ -64,18 +64,8 @@ impl ManifestAsyncModule { #[turbo_tasks::function] pub async fn manifest_chunks(self: Vc) -> Result> { let this = self.await?; - if let Some(chunk_items) = this.availability_info.available_chunk_items() { - if chunk_items - .await? - .get( - this.inner - .as_chunk_item(*ResolvedVc::upcast(this.chunking_context)) - .to_resolved() - .await?, - ) - .await? - .is_some() - { + if let Some(chunk_items) = this.availability_info.available_modules() { + if chunk_items.get(*this.inner).await?.is_some() { return Ok(Vc::cell(vec![])); } } @@ -92,7 +82,7 @@ impl ManifestAsyncModule { #[turbo_tasks::function] pub async fn content_ident(&self) -> Result> { let mut ident = self.inner.ident(); - if let Some(available_modules) = self.availability_info.available_chunk_items() { + if let Some(available_modules) = self.availability_info.available_modules() { ident = ident.with_modifier(Vc::cell(available_modules.hash().await?.to_string().into())); } diff --git a/turbopack/crates/turbopack-ecmascript/src/references/async_module.rs b/turbopack/crates/turbopack-ecmascript/src/references/async_module.rs index 74b2bcaa0309d..614a8b7d29fc3 100644 --- a/turbopack/crates/turbopack-ecmascript/src/references/async_module.rs +++ b/turbopack/crates/turbopack-ecmascript/src/references/async_module.rs @@ -10,9 +10,7 @@ use turbo_tasks::{ TryJoinIterExt, Vc, }; use turbopack_core::{ - chunk::{ - AsyncModuleInfo, ChunkableModule, ChunkableModuleReference, ChunkingContext, ChunkingType, - }, + chunk::{AsyncModuleInfo, ChunkableModuleReference, ChunkingType}, reference::{ModuleReference, ModuleReferences}, resolve::ExternalType, }; @@ -102,7 +100,6 @@ impl AsyncModule { #[turbo_tasks::function] async fn get_async_idents( &self, - chunking_context: Vc>, async_module_info: Vc, references: Vc, ) -> Result> { @@ -124,13 +121,9 @@ impl AsyncModule { } } ReferencedAsset::Some(placeable) => { - let chunk_item = placeable - .as_chunk_item(Vc::upcast(chunking_context)) - .to_resolved() - .await?; if async_module_info .referenced_async_modules - .contains(&chunk_item) + .contains(&ResolvedVc::upcast(*placeable)) { referenced_asset.get_ident().await? } else { @@ -194,14 +187,11 @@ impl AsyncModule { #[turbo_tasks::function] pub async fn code_generation( self: Vc, - chunking_context: Vc>, async_module_info: Option>, references: Vc, ) -> Result> { if let Some(async_module_info) = async_module_info { - let async_idents = self - .get_async_idents(chunking_context, async_module_info, references) - .await?; + let async_idents = self.get_async_idents(async_module_info, references).await?; if !async_idents.is_empty() { let idents = async_idents diff --git a/turbopack/crates/turbopack-ecmascript/src/references/external_module.rs b/turbopack/crates/turbopack-ecmascript/src/references/external_module.rs index 311fe65f78b3a..efa790ad6a26e 100644 --- a/turbopack/crates/turbopack-ecmascript/src/references/external_module.rs +++ b/turbopack/crates/turbopack-ecmascript/src/references/external_module.rs @@ -131,6 +131,13 @@ impl Module for CachedExternalModule { async fn references(&self) -> Result> { Ok(Vc::cell(self.additional_references.clone())) } + + #[turbo_tasks::function] + async fn is_self_async(&self) -> Result> { + Ok(Vc::cell( + self.external_type == CachedExternalType::EcmaScriptViaImport, + )) + } } #[turbo_tasks::value_impl] @@ -229,13 +236,6 @@ impl ChunkItem for CachedExternalModuleChunkItem { fn chunking_context(&self) -> Vc> { *self.chunking_context } - - #[turbo_tasks::function] - async fn is_self_async(&self) -> Result> { - Ok(Vc::cell( - self.module.await?.external_type == CachedExternalType::EcmaScriptViaImport, - )) - } } #[turbo_tasks::value_impl] diff --git a/turbopack/crates/turbopack-ecmascript/src/side_effect_optimization/facade/chunk_item.rs b/turbopack/crates/turbopack-ecmascript/src/side_effect_optimization/facade/chunk_item.rs index 6c67c8a8f9ea4..f4798c8eeba16 100644 --- a/turbopack/crates/turbopack-ecmascript/src/side_effect_optimization/facade/chunk_item.rs +++ b/turbopack/crates/turbopack-ecmascript/src/side_effect_optimization/facade/chunk_item.rs @@ -80,11 +80,11 @@ impl EcmascriptChunkItem for EcmascriptModuleFacadeChunkItem { code_gens.push(code_gen.code_generation(*chunking_context)); } } - code_gens.push(self.module.async_module().code_generation( - *chunking_context, - async_module_info, - references, - )); + code_gens.push( + self.module + .async_module() + .code_generation(async_module_info, references), + ); code_gens.push(exports.code_generation(*chunking_context)); let code_gens = code_gens.into_iter().try_join().await?; let code_gens = code_gens.iter().map(|cg| &**cg).collect::>(); @@ -150,18 +150,4 @@ impl ChunkItem for EcmascriptModuleFacadeChunkItem { fn module(&self) -> Vc> { *ResolvedVc::upcast(self.module) } - - #[turbo_tasks::function] - async fn is_self_async(&self) -> Result> { - let module = self.module; - let async_module = module.async_module(); - let references = module.references(); - let is_self_async = async_module - .resolve() - .await? - .is_self_async(references.resolve().await?) - .resolve() - .await?; - Ok(is_self_async) - } } diff --git a/turbopack/crates/turbopack-ecmascript/src/side_effect_optimization/facade/module.rs b/turbopack/crates/turbopack-ecmascript/src/side_effect_optimization/facade/module.rs index 1d038647fd356..754c71d8b7243 100644 --- a/turbopack/crates/turbopack-ecmascript/src/side_effect_optimization/facade/module.rs +++ b/turbopack/crates/turbopack-ecmascript/src/side_effect_optimization/facade/module.rs @@ -151,6 +151,19 @@ impl Module for EcmascriptModuleFacadeModule { }; Ok(Vc::cell(references)) } + + #[turbo_tasks::function] + async fn is_self_async(self: Vc) -> Result> { + let async_module = self.async_module(); + let references = self.references(); + let is_self_async = async_module + .resolve() + .await? + .is_self_async(references.resolve().await?) + .resolve() + .await?; + Ok(is_self_async) + } } #[turbo_tasks::value_impl] diff --git a/turbopack/crates/turbopack-ecmascript/src/side_effect_optimization/locals/chunk_item.rs b/turbopack/crates/turbopack-ecmascript/src/side_effect_optimization/locals/chunk_item.rs index 3542d1d556927..1e81fc17ac0d1 100644 --- a/turbopack/crates/turbopack-ecmascript/src/side_effect_optimization/locals/chunk_item.rs +++ b/turbopack/crates/turbopack-ecmascript/src/side_effect_optimization/locals/chunk_item.rs @@ -97,16 +97,4 @@ impl ChunkItem for EcmascriptModuleLocalsChunkItem { fn module(&self) -> Vc> { *ResolvedVc::upcast(self.module) } - - #[turbo_tasks::function] - async fn is_self_async(&self) -> Result> { - let module = self.module.await?; - let analyze = module.module.analyze().await?; - if let Some(async_module) = *analyze.async_module.await? { - let is_self_async = async_module.is_self_async(*analyze.local_references); - Ok(is_self_async) - } else { - Ok(Vc::cell(false)) - } - } } diff --git a/turbopack/crates/turbopack-ecmascript/src/side_effect_optimization/locals/module.rs b/turbopack/crates/turbopack-ecmascript/src/side_effect_optimization/locals/module.rs index 76d32819e2516..0890a7683d7c8 100644 --- a/turbopack/crates/turbopack-ecmascript/src/side_effect_optimization/locals/module.rs +++ b/turbopack/crates/turbopack-ecmascript/src/side_effect_optimization/locals/module.rs @@ -52,6 +52,17 @@ impl Module for EcmascriptModuleLocalsModule { let result = self.module.analyze().await?; Ok(*result.local_references) } + + #[turbo_tasks::function] + async fn is_self_async(&self) -> Result> { + let analyze = self.module.analyze().await?; + if let Some(async_module) = *analyze.async_module.await? { + let is_self_async = async_module.is_self_async(*analyze.local_references); + Ok(is_self_async) + } else { + Ok(Vc::cell(false)) + } + } } #[turbo_tasks::value_impl] diff --git a/turbopack/crates/turbopack-ecmascript/src/tree_shake/asset.rs b/turbopack/crates/turbopack-ecmascript/src/tree_shake/asset.rs index 3637dfeddc498..be6950ab1777d 100644 --- a/turbopack/crates/turbopack-ecmascript/src/tree_shake/asset.rs +++ b/turbopack/crates/turbopack-ecmascript/src/tree_shake/asset.rs @@ -274,6 +274,11 @@ impl Module for EcmascriptModulePartAsset { self.full_module.ident().with_part(*self.part) } + #[turbo_tasks::function] + fn is_self_async(self: Vc) -> Vc { + self.is_async_module() + } + #[turbo_tasks::function] async fn references(&self) -> Result> { let split_data = split_module(*self.full_module).await?; diff --git a/turbopack/crates/turbopack-ecmascript/src/tree_shake/chunk_item.rs b/turbopack/crates/turbopack-ecmascript/src/tree_shake/chunk_item.rs index 87d8025758496..b9856e1da4087 100644 --- a/turbopack/crates/turbopack-ecmascript/src/tree_shake/chunk_item.rs +++ b/turbopack/crates/turbopack-ecmascript/src/tree_shake/chunk_item.rs @@ -101,11 +101,6 @@ impl ChunkItem for EcmascriptModulePartChunkItem { fn module(&self) -> Vc> { *ResolvedVc::upcast(self.module) } - - #[turbo_tasks::function] - fn is_self_async(&self) -> Vc { - self.module.is_async_module() - } } #[turbo_tasks::value(shared)] diff --git a/turbopack/crates/turbopack-nodejs/src/ecmascript/node/content.rs b/turbopack/crates/turbopack-nodejs/src/ecmascript/node/content.rs index ae5f56537265c..1f26863e0bbf5 100644 --- a/turbopack/crates/turbopack-nodejs/src/ecmascript/node/content.rs +++ b/turbopack/crates/turbopack-nodejs/src/ecmascript/node/content.rs @@ -13,7 +13,7 @@ use turbopack_core::{ version::{Version, VersionedContent}, }; use turbopack_ecmascript::{ - chunk::{EcmascriptChunkContent, EcmascriptChunkItemExt}, + chunk::{EcmascriptChunkContent, EcmascriptChunkItemExt, EcmascriptChunkItemWithAsyncInfo}, minify::minify, utils::StringifyJs, }; @@ -52,12 +52,18 @@ pub(super) async fn chunk_items( .await? .chunk_items .iter() - .map(|&(chunk_item, async_module_info)| async move { - Ok(( - chunk_item.id().await?, - chunk_item.code(async_module_info.map(|info| *info)).await?, - )) - }) + .map( + async |&EcmascriptChunkItemWithAsyncInfo { + chunk_item, + async_info, + .. + }| { + Ok(( + chunk_item.id().await?, + chunk_item.code(async_info.map(|info| *info)).await?, + )) + }, + ) .try_join() .await } diff --git a/turbopack/crates/turbopack-nodejs/src/lib.rs b/turbopack/crates/turbopack-nodejs/src/lib.rs index 5b6808a1e8522..a86747e4506d4 100644 --- a/turbopack/crates/turbopack-nodejs/src/lib.rs +++ b/turbopack/crates/turbopack-nodejs/src/lib.rs @@ -1,3 +1,4 @@ +#![feature(async_closure)] #![feature(iter_intersperse)] #![feature(arbitrary_self_types)] #![feature(arbitrary_self_types_pointers)] diff --git a/turbopack/crates/turbopack-wasm/src/module_asset.rs b/turbopack/crates/turbopack-wasm/src/module_asset.rs index 7a9ac9799395a..92741737d5261 100644 --- a/turbopack/crates/turbopack-wasm/src/module_asset.rs +++ b/turbopack/crates/turbopack-wasm/src/module_asset.rs @@ -136,6 +136,11 @@ impl Module for WebAssemblyModuleAsset { fn references(self: Vc) -> Vc { self.loader().references() } + + #[turbo_tasks::function] + fn is_self_async(self: Vc) -> Vc { + Vc::cell(true) + } } #[turbo_tasks::value_impl] @@ -229,11 +234,6 @@ impl ChunkItem for ModuleChunkItem { fn module(&self) -> Vc> { Vc::upcast(*self.module) } - - #[turbo_tasks::function] - fn is_self_async(self: Vc) -> Vc { - Vc::cell(true) - } } #[turbo_tasks::value_impl] From 982fff339effcd2c6d021578d4280dad41e95045 Mon Sep 17 00:00:00 2001 From: Jiachi Liu Date: Sat, 11 Jan 2025 00:49:09 +0100 Subject: [PATCH 8/9] Always display version indicator (#74774) ### What Remove the condition to check when version indicator can be displayed as it doesn't have to be coupled with test mode or telemetry --- .../src/server/dev/hot-reloader-turbopack.ts | 4 +--- .../src/server/dev/hot-reloader-webpack.ts | 20 ++++--------------- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/packages/next/src/server/dev/hot-reloader-turbopack.ts b/packages/next/src/server/dev/hot-reloader-turbopack.ts index b9c6372d06084..d710bf8994a27 100644 --- a/packages/next/src/server/dev/hot-reloader-turbopack.ts +++ b/packages/next/src/server/dev/hot-reloader-turbopack.ts @@ -631,9 +631,7 @@ export async function createHotReloaderTurbopack( getNextErrorFeedbackMiddleware(opts.telemetry), ] - const versionInfoPromise = getVersionInfo( - isTestMode || opts.telemetry.isEnabled - ) + const versionInfoPromise = getVersionInfo() let devtoolsFrontendUrl: string | undefined const nodeDebugType = getNodeDebugType() diff --git a/packages/next/src/server/dev/hot-reloader-webpack.ts b/packages/next/src/server/dev/hot-reloader-webpack.ts index fcf0dcf84b69e..e71ea8a6165ff 100644 --- a/packages/next/src/server/dev/hot-reloader-webpack.ts +++ b/packages/next/src/server/dev/hot-reloader-webpack.ts @@ -86,11 +86,6 @@ import { getNodeDebugType } from '../lib/utils' import { getNextErrorFeedbackMiddleware } from '../../client/components/react-dev-overlay/server/get-next-error-feedback-middleware' const MILLISECONDS_IN_NANOSECOND = BigInt(1_000_000) -const isTestMode = !!( - process.env.NEXT_TEST_MODE || - process.env.__NEXT_TEST_MODE || - process.env.DEBUG -) function diff(a: Set, b: Set) { return new Set([...a].filter((v) => !b.has(v))) @@ -196,13 +191,9 @@ function erroredPages(compilation: webpack.Compilation) { return failedPages } -export async function getVersionInfo(enabled: boolean): Promise { +export async function getVersionInfo(): Promise { let installed = '0.0.0' - if (!enabled) { - return { installed, staleness: 'unknown' } - } - try { installed = require('next/package.json').version @@ -751,10 +742,10 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface { }) } - private async tracedGetVersionInfo(span: Span, enabled: boolean) { + private async tracedGetVersionInfo(span: Span) { const versionInfoSpan = span.traceChild('get-version-info') return versionInfoSpan.traceAsyncFn(async () => - getVersionInfo(enabled) + getVersionInfo() ) } @@ -762,10 +753,7 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface { const startSpan = this.hotReloaderSpan.traceChild('start') startSpan.stop() // Stop immediately to create an artificial parent span - this.versionInfo = await this.tracedGetVersionInfo( - startSpan, - isTestMode || this.telemetry.isEnabled - ) + this.versionInfo = await this.tracedGetVersionInfo(startSpan) const nodeDebugType = getNodeDebugType() if (nodeDebugType && !this.devtoolsFrontendUrl) { From 7fd7ee7b528bb4b855a6462c44b05434425bbcca Mon Sep 17 00:00:00 2001 From: Benjamin Woodruff Date: Fri, 10 Jan 2025 16:26:25 -0800 Subject: [PATCH 9/9] chore(turbopack): Delete some dead code that was using unresolved Vcs (#74705) --- turbopack/crates/turbopack/src/lib.rs | 75 --------------------------- 1 file changed, 75 deletions(-) diff --git a/turbopack/crates/turbopack/src/lib.rs b/turbopack/crates/turbopack/src/lib.rs index e90046a7d5605..2df20366a9dba 100644 --- a/turbopack/crates/turbopack/src/lib.rs +++ b/turbopack/crates/turbopack/src/lib.rs @@ -13,11 +13,6 @@ pub mod module_options; pub mod transition; pub(crate) mod unsupported_sass; -use std::{ - collections::{HashMap, HashSet}, - mem::swap, -}; - use anyhow::{bail, Result}; use css::{CssModuleAsset, ModuleCssAsset}; use ecmascript::{ @@ -952,76 +947,6 @@ pub async fn emit_asset_into_dir( Ok(()) } -type OutputAssetSet = HashSet>>; - -#[turbo_tasks::value(shared, local)] -struct ReferencesList { - referenced_by: HashMap>, OutputAssetSet>, -} - -#[turbo_tasks::function] -async fn compute_back_references( - aggregated: ResolvedVc, -) -> Result> { - Ok(match &*aggregated.content().await? { - &AggregatedGraphNodeContent::Asset(asset) => { - let mut referenced_by = HashMap::new(); - for &reference in asset.references().await?.iter() { - referenced_by.insert(reference, [*asset].into_iter().collect()); - } - ReferencesList { referenced_by }.into() - } - AggregatedGraphNodeContent::Children(children) => { - let mut referenced_by = - HashMap::>, OutputAssetSet>::new(); - let lists = children - .iter() - .map(|child| compute_back_references(**child)) - .collect::>(); - for list in lists { - for (key, values) in list.await?.referenced_by.iter() { - if let Some(set) = referenced_by.get_mut(key) { - for value in values { - set.insert(*value); - } - } else { - referenced_by.insert(*key, values.clone()); - } - } - } - ReferencesList { referenced_by }.into() - } - }) -} - -#[turbo_tasks::function] -async fn top_references(list: Vc) -> Result> { - let list = list.await?; - const N: usize = 5; - let mut top = Vec::<( - &ResolvedVc>, - &HashSet>>, - )>::new(); - for tuple in list.referenced_by.iter() { - let mut current = tuple; - for item in &mut top { - if item.1.len() < tuple.1.len() { - swap(item, &mut current); - } - } - if top.len() < N { - top.push(current); - } - } - Ok(ReferencesList { - referenced_by: top - .into_iter() - .map(|(asset, set)| (*asset, set.clone())) - .collect(), - } - .into()) -} - /// Replaces the externals in the result with `ExternalModuleAsset` instances. pub async fn replace_external( name: &RcStr,