-
Notifications
You must be signed in to change notification settings - Fork 16
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #704 from forcedotcom/sm/more-source-member-fields
feat: more source member fields
- Loading branch information
Showing
17 changed files
with
1,143 additions
and
1,324 deletions.
There are no files selected for viewing
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| /* | ||
| * Copyright (c) 2023, salesforce.com, inc. | ||
| * All rights reserved. | ||
| * Licensed under the BSD 3-Clause license. | ||
| * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause | ||
| */ | ||
| import fs from 'node:fs'; | ||
| import path from 'node:path'; | ||
| import { parseJsonMap } from '@salesforce/kit'; | ||
| import { lockInit, envVars as env, Logger } from '@salesforce/core'; | ||
| import { | ||
| getLegacyMetadataKey, | ||
| getMetadataKey, | ||
| getMetadataNameFromLegacyKey, | ||
| getMetadataTypeFromLegacyKey, | ||
| } from '../functions'; | ||
| import { RemoteChangeElement } from '../types'; | ||
| import { ContentsV0, ContentsV1, MemberRevision, MemberRevisionLegacy } from './types'; | ||
|
|
||
| export const FILENAME = 'maxRevision.json'; | ||
|
|
||
| export const getFilePath = (orgId: string): string => path.join('.sf', 'orgs', orgId, FILENAME); | ||
|
|
||
| export const readFileContents = async (filePath: string): Promise<ContentsV1 | Record<string, never>> => { | ||
| try { | ||
| const contents = await fs.promises.readFile(filePath, 'utf8'); | ||
| const parsedContents = parseJsonMap<ContentsV1 | ContentsV0>(contents, filePath); | ||
| if (parsedContents.fileVersion === 1) { | ||
| return parsedContents; | ||
| } | ||
| Logger.childFromRoot('remoteSourceTrackingService:readFileContents').debug( | ||
| `older tracking file version, found ${ | ||
| parsedContents.fileVersion ?? 'undefined' | ||
| }. Upgrading file contents. Some expected data may be missing` | ||
| ); | ||
| return upgradeFileContents(parsedContents); | ||
| } catch (e) { | ||
| Logger.childFromRoot('remoteSourceTrackingService:readFileContents').debug( | ||
| `Error reading or parsing file file at ${filePath}. Will treat as an empty file.`, | ||
| e | ||
| ); | ||
|
|
||
| return {}; | ||
| } | ||
| }; | ||
|
|
||
| export const revisionToRemoteChangeElement = (memberRevision: MemberRevision): RemoteChangeElement => ({ | ||
| type: memberRevision.MemberType, | ||
| name: memberRevision.MemberName, | ||
| deleted: memberRevision.IsNameObsolete, | ||
| modified: memberRevision.IsNewMember === false, | ||
| revisionCounter: memberRevision.RevisionCounter, | ||
| changedBy: memberRevision.ChangedBy, | ||
| memberIdOrName: memberRevision.MemberIdOrName, | ||
| }); | ||
|
|
||
| export const upgradeFileContents = (contents: ContentsV0): ContentsV1 => ({ | ||
| fileVersion: 1, | ||
| serverMaxRevisionCounter: contents.serverMaxRevisionCounter, | ||
| // @ts-expect-error the old file didn't store the IsNewMember field or any indication of whether the member was add/modified | ||
| sourceMembers: Object.fromEntries( | ||
| // it's the old version | ||
| Object.entries(contents.sourceMembers).map(([key, value]) => [ | ||
| getMetadataKey(getMetadataTypeFromLegacyKey(key), getMetadataNameFromLegacyKey(key)), | ||
| { | ||
| MemberName: getMetadataNameFromLegacyKey(key), | ||
| MemberType: value.memberType, | ||
| IsNameObsolete: value.isNameObsolete, | ||
| RevisionCounter: value.serverRevisionCounter, | ||
| lastRetrievedFromServer: value.lastRetrievedFromServer ?? undefined, | ||
| ChangedBy: 'unknown', | ||
| MemberIdOrName: 'unknown', | ||
| }, | ||
| ]) | ||
| ), | ||
| }); | ||
|
|
||
| export const writeTrackingFile = async ({ | ||
| filePath, | ||
| maxCounter, | ||
| members, | ||
| }: { | ||
| filePath: string; | ||
| maxCounter: number; | ||
| members: Map<string, MemberRevision>; | ||
| }): Promise<void> => { | ||
| const lockResult = await lockInit(filePath); | ||
| const CURRENT_FILE_VERSION_ENV = env.getNumber('SF_SOURCE_TRACKING_FILE_VERSION') ?? 0; | ||
| const contents = | ||
| CURRENT_FILE_VERSION_ENV === 1 | ||
| ? ({ | ||
| fileVersion: 1, | ||
| serverMaxRevisionCounter: maxCounter, | ||
| sourceMembers: Object.fromEntries(members), | ||
| } satisfies ContentsV1) | ||
| : ({ | ||
| fileVersion: 0, | ||
| serverMaxRevisionCounter: maxCounter, | ||
| sourceMembers: Object.fromEntries(Array.from(members.entries()).map(toLegacyMemberRevision)), | ||
| } satisfies ContentsV0); | ||
| await lockResult.writeAndUnlock(JSON.stringify(contents, null, 4)); | ||
| }; | ||
|
|
||
| export const toLegacyMemberRevision = ([, member]: [string, MemberRevision]): [key: string, MemberRevisionLegacy] => [ | ||
| getLegacyMetadataKey(member.MemberType, member.MemberName), | ||
| { | ||
| memberType: member.MemberType, | ||
| serverRevisionCounter: member.RevisionCounter, | ||
| lastRetrievedFromServer: member.lastRetrievedFromServer ?? null, | ||
| isNameObsolete: member.IsNameObsolete, | ||
| }, | ||
| ]; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
| /* | ||
| * Copyright (c) 2023, salesforce.com, inc. | ||
| * All rights reserved. | ||
| * Licensed under the BSD 3-Clause license. | ||
| * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause | ||
| */ | ||
| import { Connection, envVars as env, SfError, trimTo15 } from '@salesforce/core'; | ||
| import { Duration } from '@salesforce/kit'; | ||
| import { PinoLogger } from './remoteSourceTrackingService'; | ||
| import { SOURCE_MEMBER_FIELDS, SourceMember } from './types'; | ||
|
|
||
| export const calculateTimeout = | ||
| (logger: PinoLogger) => | ||
| (memberCount: number): Duration => { | ||
| const overriddenTimeout = env.getNumber('SF_SOURCE_MEMBER_POLLING_TIMEOUT', 0); | ||
| if (overriddenTimeout > 0) { | ||
| logger.debug(`Overriding SourceMember polling timeout to ${overriddenTimeout}`); | ||
| return Duration.seconds(overriddenTimeout); | ||
| } | ||
|
|
||
| // Calculate a polling timeout for SourceMembers based on the number of | ||
| // member names being polled plus a buffer of 5 seconds. This will | ||
| // wait 50s for each 1000 components, plus 5s. | ||
| const pollingTimeout = Math.ceil(memberCount * 0.05) + 5; | ||
| logger.debug(`Computed SourceMember polling timeout of ${pollingTimeout}s`); | ||
| return Duration.seconds(pollingTimeout); | ||
| }; | ||
| /** exported only for spy/mock */ | ||
|
|
||
| export const querySourceMembersTo = async (conn: Connection, toRevision: number): Promise<SourceMember[]> => { | ||
| const query = `SELECT ${SOURCE_MEMBER_FIELDS.join(', ')} FROM SourceMember WHERE RevisionCounter <= ${toRevision}`; | ||
| return queryFn(conn, query); | ||
| }; | ||
|
|
||
| export const querySourceMembersFrom = async ({ | ||
| conn, | ||
| fromRevision, | ||
| queryCache, | ||
| userQueryCache, | ||
| logger, | ||
| }: { | ||
| conn: Connection; | ||
| fromRevision: number; | ||
| /** optional cache, used if present. Side effect: cache will be mutated */ | ||
| queryCache?: Map<number, SourceMember[]>; | ||
| /** optional cache, used if present. Side effect: cache will be mutated */ | ||
| userQueryCache?: Map<string, string>; | ||
| /** if you don't pass in a logger, you get no log output */ | ||
| logger?: PinoLogger; | ||
| }): Promise<SourceMember[]> => { | ||
| if (queryCache) { | ||
| // Check cache first and return if found. | ||
| const cachedQueryResult = queryCache.get(fromRevision); | ||
| if (cachedQueryResult) { | ||
| logger?.debug(`Using cache for SourceMember query for revision ${fromRevision}`); | ||
| return cachedQueryResult; | ||
| } | ||
| } | ||
|
|
||
| // because `serverMaxRevisionCounter` is always updated, we need to select > to catch the most recent change | ||
| const query = `SELECT ${SOURCE_MEMBER_FIELDS.join(', ')} FROM SourceMember WHERE RevisionCounter > ${fromRevision}`; | ||
| logger?.debug(`Query: ${query}`); | ||
|
|
||
| const queryResult = await queryFn(conn, query); | ||
| if (userQueryCache) { | ||
| await updateCacheWithUnknownUsers(conn, queryResult, userQueryCache); | ||
| } | ||
| const queryResultWithResolvedUsers = queryResult.map((member) => ({ | ||
| ...member, | ||
| ChangedBy: userQueryCache?.get(member.ChangedBy) ?? member.ChangedBy, | ||
| })); | ||
| queryCache?.set(fromRevision, queryResultWithResolvedUsers); | ||
|
|
||
| return queryResultWithResolvedUsers; | ||
| }; | ||
|
|
||
| export const queryFn = async (conn: Connection, query: string): Promise<SourceMember[]> => { | ||
| try { | ||
| return (await conn.tooling.query<SourceMember>(query, { autoFetch: true, maxFetch: 50_000 })).records.map( | ||
| sourceMemberCorrections | ||
| ); | ||
| } catch (error) { | ||
| throw SfError.wrap(error); | ||
| } | ||
| }; | ||
|
|
||
| /** A series of workarounds for server-side bugs. Each bug should be filed against a team, with a WI, so we know when these are fixed and can be removed */ | ||
| const sourceMemberCorrections = (sourceMember: SourceMember): SourceMember => { | ||
| if (sourceMember.MemberType === 'QuickActionDefinition') { | ||
| return { ...sourceMember, MemberType: 'QuickAction' }; // W-15837125 | ||
| } | ||
| return sourceMember; | ||
| }; | ||
|
|
||
| const updateCacheWithUnknownUsers = async ( | ||
| conn: Connection, | ||
| queryResult: SourceMember[], | ||
| userCache: Map<string, string> | ||
| ): Promise<void> => { | ||
| const unknownUsers = new Set<string>(queryResult.map((member) => member.ChangedBy).filter((u) => !userCache.has(u))); | ||
| if (unknownUsers.size > 0) { | ||
| const userQuery = `SELECT Id, Name FROM User WHERE Id IN ('${Array.from(unknownUsers).join("','")}')`; | ||
| (await conn.query<{ Id: string; Name: string }>(userQuery, { autoFetch: true, maxFetch: 50_000 })).records.map( | ||
| (u) => userCache.set(trimTo15(u.Id), u.Name) | ||
| ); | ||
| } | ||
| }; |
Oops, something went wrong.