diff --git a/src/collections/componentSetBuilder.ts b/src/collections/componentSetBuilder.ts index 578d5f06f..3e49c0351 100644 --- a/src/collections/componentSetBuilder.ts +++ b/src/collections/componentSetBuilder.ts @@ -29,8 +29,18 @@ export type ManifestOption = { }; type MetadataOption = { + /** + * Array of metadata type:name pairs to include in the ComponentSet. + */ metadataEntries: string[]; + /** + * Array of filesystem directory paths to search for local metadata to include in the ComponentSet. + */ directoryPaths: string[]; + /** + * Array of metadata type:name pairs to exclude from the ComponentSet. + */ + excludedEntries?: string[]; /** * Array of metadata type:name pairs to delete before the deploy. Use of wildcards is not allowed. */ @@ -60,6 +70,14 @@ export type ComponentSetOptions = { type MetadataMap = Map; +let logger: Logger; +const getLogger = (): Logger => { + if (!logger) { + logger = Logger.childFromRoot('ComponentSetBuilder'); + } + return logger; +}; + export class ComponentSetBuilder { /** * Builds a ComponentSet that can be used for source conversion, @@ -71,14 +89,13 @@ export class ComponentSetBuilder { */ public static async build(options: ComponentSetOptions): Promise { - const logger = Logger.childFromRoot('componentSetBuilder'); let componentSet: ComponentSet | undefined; const { sourcepath, manifest, metadata, packagenames, org } = options; const registry = new RegistryAccess(undefined, options.projectDir); if (sourcepath) { - logger.debug(`Building ComponentSet from sourcepath: ${sourcepath.join(', ')}`); + getLogger().debug(`Building ComponentSet from sourcepath: ${sourcepath.join(', ')}`); const fsPaths = sourcepath.map(validateAndResolvePath); componentSet = ComponentSet.fromSource({ fsPaths, @@ -88,16 +105,16 @@ export class ComponentSetBuilder { // Return empty ComponentSet and use packageNames in the connection via `.retrieve` options if (packagenames) { - logger.debug(`Building ComponentSet for packagenames: ${packagenames.toString()}`); + getLogger().debug(`Building ComponentSet for packagenames: ${packagenames.toString()}`); componentSet ??= new ComponentSet(undefined, registry); } // Resolve manifest with source in package directories. if (manifest) { - logger.debug(`Building ComponentSet from manifest: ${manifest.manifestPath}`); + getLogger().debug(`Building ComponentSet from manifest: ${manifest.manifestPath}`); assertFileExists(manifest.manifestPath); - logger.debug(`Searching in packageDir: ${manifest.directoryPaths.join(', ')} for matching metadata`); + getLogger().debug(`Searching in packageDir: ${manifest.directoryPaths.join(', ')} for matching metadata`); componentSet = await ComponentSet.fromManifest({ manifestPath: manifest.manifestPath, resolveSourcePaths: manifest.directoryPaths, @@ -108,9 +125,10 @@ export class ComponentSetBuilder { }); } - // Resolve metadata entries with source in package directories. - if (metadata) { - logger.debug(`Building ComponentSet from metadata: ${metadata.metadataEntries.toString()}`); + // Resolve metadata entries with source in package directories, unless we are building a ComponentSet + // from metadata in an org. + if (metadata && !org) { + getLogger().debug(`Building ComponentSet from metadata: ${metadata.metadataEntries.toString()}`); const directoryPaths = metadata.directoryPaths; componentSet ??= new ComponentSet(undefined, registry); const componentSetFilter = new ComponentSet(undefined, registry); @@ -122,7 +140,7 @@ export class ComponentSetBuilder { .map(addToComponentSet(componentSet)) .map(addToComponentSet(componentSetFilter)); - logger.debug(`Searching for matching metadata in directories: ${directoryPaths.join(', ')}`); + getLogger().debug(`Searching for matching metadata in directories: ${directoryPaths.join(', ')}`); // add destructive changes if defined. Because these are deletes, all entries // are resolved to SourceComponents @@ -170,25 +188,8 @@ export class ComponentSetBuilder { // Resolve metadata entries with an org connection if (org) { componentSet ??= new ComponentSet(undefined, registry); - - logger.debug( - `Building ComponentSet from targetUsername: ${org.username} ${ - metadata ? `filtered by metadata: ${metadata.metadataEntries.toString()}` : '' - }` - ); - - const mdMap = metadata - ? buildMapFromComponents(metadata.metadataEntries.map(entryToTypeAndName(registry))) - : (new Map() as MetadataMap); - - const fromConnection = await ComponentSet.fromConnection({ - usernameOrConnection: (await StateAggregator.getInstance()).aliases.getUsername(org.username) ?? org.username, - componentFilter: getOrgComponentFilter(org, mdMap, metadata), - metadataTypes: mdMap.size ? Array.from(mdMap.keys()) : undefined, - registry, - }); - - fromConnection.toArray().map(addToComponentSet(componentSet)); + const orgComponentSet = await this.resolveOrgComponents(registry, org, metadata); + orgComponentSet.toArray().map(addToComponentSet(componentSet)); } // there should have been a componentSet created by this point. @@ -197,9 +198,35 @@ export class ComponentSetBuilder { componentSet.sourceApiVersion ??= options.sourceapiversion; componentSet.projectDirectory = options.projectDir; - logComponents(logger, componentSet); + logComponents(componentSet); return componentSet; } + + private static async resolveOrgComponents( + registry: RegistryAccess, + org: OrgOption, + metadata?: MetadataOption + ): Promise { + let mdMap = new Map() as MetadataMap; + let debugMsg = `Building ComponentSet from metadata in an org using targetUsername: ${org.username}`; + if (metadata) { + if (metadata.metadataEntries?.length) { + debugMsg += ` filtering on metadata: ${metadata.metadataEntries.toString()}`; + } + if (metadata.excludedEntries?.length) { + debugMsg += ` excluding metadata: ${metadata.excludedEntries.toString()}`; + } + mdMap = buildMapFromMetadata(metadata, registry); + } + getLogger().debug(debugMsg); + + return ComponentSet.fromConnection({ + usernameOrConnection: (await StateAggregator.getInstance()).aliases.getUsername(org.username) ?? org.username, + componentFilter: getOrgComponentFilter(org, mdMap, metadata), + metadataTypes: mdMap.size ? Array.from(mdMap.keys()) : undefined, + registry, + }); + } } const addToComponentSet = @@ -234,19 +261,19 @@ const assertNoWildcardInDestructiveEntries = (mdEntry: MetadataTypeAndMetadataNa /** This is only for debug output of matched files based on the command flags. * It will log up to 20 file matches. */ -const logComponents = (logger: Logger, componentSet: ComponentSet): void => { - logger.debug(`Matching metadata files (${componentSet.size}):`); +const logComponents = (componentSet: ComponentSet): void => { + getLogger().debug(`Matching metadata files (${componentSet.size}):`); const components = componentSet.getSourceComponents().toArray(); components .slice(0, 20) .map((cmp) => cmp.content ?? cmp.xml ?? cmp.fullName) - .map((m) => logger.debug(m)); - if (components.length > 20) logger.debug(`(showing 20 of ${componentSet.size} matches)`); + .map((m) => getLogger().debug(m)); + if (components.length > 20) getLogger().debug(`(showing 20 of ${componentSet.size} matches)`); - logger.debug(`ComponentSet apiVersion = ${componentSet.apiVersion ?? ''}`); - logger.debug(`ComponentSet sourceApiVersion = ${componentSet.sourceApiVersion ?? ''}`); + getLogger().debug(`ComponentSet apiVersion = ${componentSet.apiVersion ?? ''}`); + getLogger().debug(`ComponentSet sourceApiVersion = ${componentSet.sourceApiVersion ?? ''}`); }; const getOrgComponentFilter = ( @@ -254,7 +281,7 @@ const getOrgComponentFilter = ( mdMap: MetadataMap, metadata?: MetadataOption ): FromConnectionOptions['componentFilter'] => - metadata + metadata?.metadataEntries?.length ? (component: Partial): boolean => { if (component.type && component.fullName) { const mdMapEntry = mdMap.get(component.type); @@ -312,11 +339,33 @@ const typeAndNameToMetadataComponents = .filter((cs) => minimatch(cs.fullName, metadataName)) : [{ type, fullName: metadataName }]; -// TODO: use Map.groupBy when it's available -const buildMapFromComponents = (components: MetadataTypeAndMetadataName[]): MetadataMap => { +const buildMapFromMetadata = (mdOption: MetadataOption, registry: RegistryAccess): MetadataMap => { const mdMap: MetadataMap = new Map(); - components.map((cmp) => { - mdMap.set(cmp.type.name, [...(mdMap.get(cmp.type.name) ?? []), cmp.metadataName]); - }); + + // Add metadata type entries we were told to include + if (mdOption.metadataEntries?.length) { + mdOption.metadataEntries.map(entryToTypeAndName(registry)).map((cmp) => { + mdMap.set(cmp.type.name, [...(mdMap.get(cmp.type.name) ?? []), cmp.metadataName]); + }); + } + + // Build an array of excluded types from the options + if (mdOption.excludedEntries?.length) { + const excludedTypes: string[] = []; + mdOption.excludedEntries.map(entryToTypeAndName(registry)).map((cmp) => { + if (cmp.metadataName === '*') { + excludedTypes.push(cmp.type.name); + } + }); + if (mdMap.size === 0) { + // we are excluding specific metadata types from all supported types + Object.values(registry.getRegistry().types).map((t) => { + if (!excludedTypes.includes(t.name)) { + mdMap.set(t.name, []); + } + }); + } + } + return mdMap; }; diff --git a/src/resolve/connectionResolver.ts b/src/resolve/connectionResolver.ts index 3109b5880..be3f814ab 100644 --- a/src/resolve/connectionResolver.ts +++ b/src/resolve/connectionResolver.ts @@ -5,8 +5,10 @@ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ +import { inspect } from 'node:util'; import { Connection, Logger, Messages, Lifecycle, SfError } from '@salesforce/core'; import { ensurePlainObject, ensureString, isPlainObject } from '@salesforce/ts-types'; +import { env } from '@salesforce/kit'; import { RegistryAccess } from '../registry/registryAccess'; import { MetadataType } from '../registry/types'; import { standardValueSet } from '../registry/standardvalueset'; @@ -24,8 +26,30 @@ export type ResolveConnectionResult = { apiVersion: string; }; +let requestCount = 0; +let shouldQueryStandardValueSets = false; + +let logger: Logger; +const getLogger = (): Logger => { + if (!logger) { + logger = Logger.childFromRoot('ConnectionResolver'); + } + return logger; +}; + +// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** +// +// NOTE: The `listMetadata` API supports passing 3 metadata types per call but we +// can't do this because if 1 of the 3 types is not supported by the org (or +// errors in some way) we don't get any data back about the other types. This +// means we are forced to make listMetadata calls for individual metadata types. +// +// *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** + /** - * Resolve MetadataComponents from an org connection + * Resolve MetadataComponents from an org connection by making listMetadata API calls + * for the specified metadata types (`mdTypes` arg) or all supported metadata types + * in the registry. */ export class ConnectionResolver { private connection: Connection; @@ -35,6 +59,8 @@ export class ConnectionResolver { // all types defined in the registry. private mdTypeNames: string[]; + private requestBatchSize: number; + public constructor(connection: Connection, registry = new RegistryAccess(), mdTypes?: string[]) { this.connection = connection; this.registry = registry; @@ -42,21 +68,28 @@ export class ConnectionResolver { ? // ensure the types passed in are valid per the registry mdTypes.filter((t) => this.registry.getTypeByName(t)) : Object.values(this.registry.getRegistry().types).map((t) => t.name); + + // To limit the number of concurrent requests, batch them per an env var. + // By default there is no batching. + this.requestBatchSize = env.getNumber('SF_LIST_METADATA_BATCH_SIZE', -1); } public async resolve( componentFilter = (component: Partial): boolean => isPlainObject(component) ): Promise { + // Aggregate array of metadata records in the org const Aggregator: Array> = []; - const childrenPromises: Array> = []; - const componentTypes: Set = new Set(); + // Folder component type names. Each array value has the form [type::folder] + const folderComponentTypes: string[] = []; + // Child component type names + const childComponentTypes: Set = new Set(); + const lifecycle = Lifecycle.getInstance(); - const componentFromDescribe = ( - await Promise.all(this.mdTypeNames.map((type) => listMembers(this.registry)(this.connection)({ type }))) - ).flat(); + // Make batched listMetadata requests for top level metadata + const listMetadataResponses = await this.sendBatchedRequests(this.mdTypeNames); - for (const component of componentFromDescribe) { + for (const component of listMetadataResponses) { let componentType: MetadataType; if (isNonEmptyString(component.type)) { componentType = this.registry.getTypeByName(component.type); @@ -86,30 +119,34 @@ export class ConnectionResolver { } Aggregator.push(component); - componentTypes.add(componentType); if (componentType.folderContentType) { - childrenPromises.push( - listMembers(this.registry)(this.connection)({ - type: this.registry.getTypeByName(componentType.folderContentType).name, - folder: component.fullName, - }) - ); + const type = this.registry.getTypeByName(componentType.folderContentType).name; + const folder = component.fullName; + folderComponentTypes.push(`${type}::${folder}`); } - } - for (const componentType of componentTypes) { const childTypes = componentType.children?.types; if (childTypes) { - Object.values(childTypes).map((childType) => { - childrenPromises.push(listMembers(this.registry)(this.connection)({ type: childType.name })); - }); + Object.values(childTypes).map((childType) => childComponentTypes.add(childType.name)); } } - for await (const childrenResult of childrenPromises) { - Aggregator.push(...childrenResult); + if (folderComponentTypes.length) { + Aggregator.push(...(await this.sendBatchedRequests(folderComponentTypes))); + } + + if (childComponentTypes.size > 0) { + Aggregator.push(...(await this.sendBatchedRequests(Array.from(childComponentTypes)))); } + // If we need to query the list of StandardValueSets (i.e., it's included in this.mdTypeNames) + // make those requests now. + if (shouldQueryStandardValueSets) { + Aggregator.push(...(await this.sendBatchedQueries())); + } + + getLogger().debug(`https request count = ${requestCount}`); + return { components: Aggregator.filter(componentFilter).map((component) => ({ fullName: ensureString( @@ -128,50 +165,126 @@ export class ConnectionResolver { apiVersion: this.connection.getApiVersion(), }; } + + // Send batched listMetadata requests based on the SF_LIST_METADATA_BATCH_SIZE env var. + private async sendBatchedRequests(listMdQueries: string[]): Promise { + const listMetadataResponses: RelevantFileProperties[] = []; + let listMetadataRequests: Array> = []; + + const sendIt = async (): Promise => { + const requestBatch = (await Promise.all(listMetadataRequests)).flat(); + listMetadataResponses.push(...requestBatch); + }; + + // Make batched listMetadata requests + for (let i = 0; i < listMdQueries.length; i++) { + const q = listMdQueries[i].split('::'); + const listMdQuery = { type: q[0] } as ListMetadataQuery; + if (q[1]) { + listMdQuery.folder = q[1]; + } + listMetadataRequests.push(listMembers(this.registry)(this.connection)(listMdQuery)); + if (this.requestBatchSize > 0 && i !== 0 && i % this.requestBatchSize === 0) { + getLogger().debug(`Awaiting listMetadata requests ${i - this.requestBatchSize + 1} - ${i}`); + // We are deliberately awaiting the results of batches to throttle requests. + // eslint-disable-next-line no-await-in-loop + await sendIt(); + // Reset the requests for the next batch + listMetadataRequests = []; + } + + // Always flush the last batch; or send non-batched requests + if (i === listMdQueries.length - 1) { + getLogger().debug('Awaiting listMetadata requests'); + // We are deliberately awaiting the results of batches to throttle requests. + // eslint-disable-next-line no-await-in-loop + await sendIt(); + } + } + return listMetadataResponses; + } + + // Send batched queries for a known subset of StandardValueSets based on the + // SF_LIST_METADATA_BATCH_SIZE env var. + private async sendBatchedQueries(): Promise { + const mdType = this.registry.getTypeByName('StandardValueSet'); + const queryResponses: RelevantFileProperties[] = []; + let queryRequests: Array> = []; + + const sendIt = async (): Promise => { + const requestBatch = (await Promise.all(queryRequests)).flat(); + queryResponses.push(...requestBatch.filter((rb) => !!rb)); + }; + + // Make batched query requests + const svsNames = standardValueSet.fullnames; + for (let i = 0; i < svsNames.length; i++) { + const svsFullName = svsNames[i]; + queryRequests.push(querySvs(this.connection)(svsFullName, mdType)); + if (this.requestBatchSize > 0 && i !== 0 && i % this.requestBatchSize === 0) { + getLogger().debug(`Awaiting StandardValueSet queries ${i - this.requestBatchSize + 1} - ${i}`); + // We are deliberately awaiting the results of batches to throttle requests. + // eslint-disable-next-line no-await-in-loop + await sendIt(); + // Reset the requests for the next batch + queryRequests = []; + } + + // Always flush the last batch; or send non-batched requests + if (i === svsNames.length - 1) { + getLogger().debug('Awaiting StandardValueSet queries'); + // We are deliberately awaiting the results of batches to throttle requests. + // eslint-disable-next-line no-await-in-loop + await sendIt(); + } + } + return queryResponses; + } } +const querySvs = + (connection: Connection) => + async (svsFullName: string, svsType: MetadataType): Promise => { + try { + requestCount++; + getLogger().debug(`StandardValueSet query for ${svsFullName}`); + const standardValueSetRecord: StdValueSetRecord = await connection.singleRecordQuery( + `SELECT Id, MasterLabel, Metadata FROM StandardValueSet WHERE MasterLabel = '${svsFullName}'`, + { tooling: true } + ); + if (standardValueSetRecord.Metadata.standardValue.length) { + return { + fullName: standardValueSetRecord.MasterLabel, + fileName: `${svsType.directoryName}/${standardValueSetRecord.MasterLabel}.${svsType.suffix ?? ''}`, + type: svsType.name, + }; + } + } catch (error) { + const err = SfError.wrap(error); + getLogger().debug(`[${svsFullName}] ${err.message}`); + } + }; + const listMembers = (registry: RegistryAccess) => (connection: Connection) => async (query: ListMetadataQuery): Promise => { const mdType = registry.getTypeByName(query.type); - // Workaround because metadata.list({ type: 'StandardValueSet' }) returns [] + // Workaround because metadata.list({ type: 'StandardValueSet' }) returns []. + // Query for a subset of known StandardValueSets after all listMetadata calls. if (mdType.name === registry.getRegistry().types.standardvalueset.name) { - const members: RelevantFileProperties[] = []; - - const standardValueSetPromises = standardValueSet.fullnames.map(async (standardValueSetFullName) => { - try { - const standardValueSetRecord: StdValueSetRecord = await connection.singleRecordQuery( - `SELECT Id, MasterLabel, Metadata FROM StandardValueSet WHERE MasterLabel = '${standardValueSetFullName}'`, - { tooling: true } - ); - - return ( - standardValueSetRecord.Metadata.standardValue.length && { - fullName: standardValueSetRecord.MasterLabel, - fileName: `${mdType.directoryName}/${standardValueSetRecord.MasterLabel}.${mdType.suffix ?? ''}`, - type: mdType.name, - } - ); - } catch (err) { - const logger = Logger.childFromRoot('ConnectionResolver.listMembers'); - logger.debug(err); - } - }); - for await (const standardValueSetResult of standardValueSetPromises) { - if (standardValueSetResult) { - members.push(standardValueSetResult); - } - } - return members; + shouldQueryStandardValueSets = true; + return []; } try { + requestCount++; + getLogger().debug(`listMetadata for ${inspect(query)}`); return (await connection.metadata.list(query)).map(inferFilenamesFromType(mdType)); } catch (error) { - const logger = Logger.childFromRoot('ConnectionResolver.listMembers'); - logger.debug((error as Error).message); + const err = SfError.wrap(error); + getLogger().debug(`[${mdType.name}] ${err.message}`); return []; } }; diff --git a/test/collections/componentSetBuilder.test.ts b/test/collections/componentSetBuilder.test.ts index 14beb029f..2147b09c4 100644 --- a/test/collections/componentSetBuilder.test.ts +++ b/test/collections/componentSetBuilder.test.ts @@ -486,10 +486,10 @@ describe('ComponentSetBuilder', () => { }; const compSet = await ComponentSetBuilder.build(options); - expect(fromSourceStub.callCount).to.equal(2); + expect(fromSourceStub.callCount).to.equal(0); expect(fromConnectionStub.callCount).to.equal(1); - expect(compSet.size).to.equal(2); - expect(compSet.has(apexClassComponent)).to.equal(true); + expect(compSet.size).to.equal(1); + expect(compSet.has(apexClassComponent)).to.equal(false); expect(compSet.has(apexClassWildcardMatch)).to.equal(true); });