diff --git a/messages/resources.json b/messages/resources.json index 97bb8c24..da4eefda 100644 --- a/messages/resources.json +++ b/messages/resources.json @@ -168,6 +168,8 @@ "mappingSourceRecords": "{%s} Outbound mapping of records to %s ...", "mappingTargetRecords": "{%s} Inbound mapping of records from %s ...", "retrievingData": "Fetching the data (%s) ...", + "usingRestApi": "{%s} Using REST API to retrieve the data ...", + "usingBulkAPIQuery": "{%s} Using Bulk API Query to retrieve the data ...", "retrievingDataCompleted": "Data retrieval (%s) has been completed.", "queryingAll": "{%s} Fetching the %s data from %s (%s: all records) ...", "queryingIn": "{%s} Fetching the %s data from %s (%s: filtered records) ...", diff --git a/src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonScript.ts b/src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonScript.ts index d29e1d76..5756fca2 100644 --- a/src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonScript.ts +++ b/src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonScript.ts @@ -25,6 +25,7 @@ export default interface ISfdmuRunCustomAddonScript { pollingQueryTimeoutMs: number; concurrencyMode: "Serial" | "Parallel"; bulkThreshold: number; + queryBulkApiThreshold: number; bulkApiVersion: string; bulkApiV1BatchSize: number; restApiBatchSize: number; diff --git a/src/modules/components/common_components/logger.ts b/src/modules/components/common_components/logger.ts index 03337437..407d1616 100644 --- a/src/modules/components/common_components/logger.ts +++ b/src/modules/components/common_components/logger.ts @@ -190,6 +190,8 @@ export enum RESOURCES { mappingTargetRecords = "mappingTargetRecords", retrievingData = "retrievingData", retrievingDataCompleted = "retrievingDataCompleted", + usingRestApi = "usingRestApi", + usingBulkAPIQuery = "usingBulkAPIQuery", queryingAll = "queryingAll", queryingIn = "queryingIn", queryingIn2 = "queryingIn2", diff --git a/src/modules/components/common_components/sfdx.ts b/src/modules/components/common_components/sfdx.ts index 874d4b2b..4cba8fa3 100644 --- a/src/modules/components/common_components/sfdx.ts +++ b/src/modules/components/common_components/sfdx.ts @@ -71,7 +71,7 @@ export class Sfdx implements IAppSfdxService, IFieldMapping { * Handles composite fields. * * @param {string} soql The SOQL query - * @param {boolean} useBulkQueryApi true to use Bulk Query Api instead of the Collection Api + * @param {boolean} useBulkQueryApi true to use Bulk Query Api instead of the Rest Api * @param {boolean} bulkQueryPollTimeout If set and useBuilkQueryApi is true, polling for the query results will timeout after this amount of milliseconds * @returns {Promise>} * @memberof ApiSf @@ -199,6 +199,13 @@ export class Sfdx implements IAppSfdxService, IFieldMapping { let parsedQuery = parseQuery(soql); soql = this.sourceQueryToTarget(soql, parsedQuery.sObject).query; + if (!useBulkQueryApi) { + this.logger.infoVerbose(RESOURCES.usingRestApi, parsedQuery.sObject); + } else { + this.logger.infoVerbose(RESOURCES.usingBulkAPIQuery, parsedQuery.sObject); + } + + // Query records ///// let records = [].concat(await ___queryAsync(soql)); if (/FROM Group([\s]+|$)/i.test(soql)) { diff --git a/src/modules/models/common_models/helpers_classes.ts b/src/modules/models/common_models/helpers_classes.ts index 126749a5..b0578a3d 100644 --- a/src/modules/models/common_models/helpers_classes.ts +++ b/src/modules/models/common_models/helpers_classes.ts @@ -341,8 +341,10 @@ export class TaskOrgData { * @memberof TaskOrgData */ get useBulkQueryApi(): boolean { - return this.isSource ? this.task.sourceTotalRecorsCount > CONSTANTS.QUERY_BULK_API_THRESHOLD : - this.task.targetTotalRecorsCount > CONSTANTS.QUERY_BULK_API_THRESHOLD; + const bulkThreshold = this.task.script.queryBulkApiThreshold || CONSTANTS.QUERY_BULK_API_THRESHOLD; + return this.isSource ? this.task.sourceTotalRecorsCount >= bulkThreshold : + this.task.targetTotalRecorsCount >= bulkThreshold; + } /** diff --git a/src/modules/models/script_models/script.ts b/src/modules/models/script_models/script.ts index 839c81e4..d02c2e3b 100644 --- a/src/modules/models/script_models/script.ts +++ b/src/modules/models/script_models/script.ts @@ -78,6 +78,7 @@ export default class Script implements IAppScript, ISfdmuRunScript { pollingQueryTimeoutMs: number = CONSTANTS.DEFAULT_POLLING_QUERY_TIMEOUT_MS; concurrencyMode: "Serial" | "Parallel" = "Parallel"; bulkThreshold: number = CONSTANTS.DEFAULT_BULK_API_THRESHOLD_RECORDS; + queryBulkApiThreshold: number = CONSTANTS.QUERY_BULK_API_THRESHOLD; bulkApiVersion: string = CONSTANTS.DEFAULT_BULK_API_VERSION; bulkApiV1BatchSize: number; restApiBatchSize: number;