feat: Added the queryBulkApiThreshold parameter

forcedotcom · Nov 13, 2023 · 12a6518 · 12a6518
1 parent a8c0cd2
commit 12a6518
Show file tree

Hide file tree

Showing 6 changed files with 18 additions and 3 deletions.
diff --git a/messages/resources.json b/messages/resources.json
@@ -168,6 +168,8 @@
     "mappingSourceRecords": "{%s} Outbound mapping of records to %s ...",
     "mappingTargetRecords": "{%s} Inbound mapping of records from %s ...",
     "retrievingData": "Fetching the data (%s) ...",
+    "usingRestApi": "{%s} Using REST API to retrieve the data ...",
+    "usingBulkAPIQuery": "{%s} Using Bulk API Query to retrieve the data ...",
     "retrievingDataCompleted": "Data retrieval (%s) has been completed.",
     "queryingAll": "{%s} Fetching the %s data from %s (%s: all records) ...",
     "queryingIn": "{%s} Fetching the %s data from %s (%s: filtered records) ...",

diff --git a/src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonScript.ts b/src/addons/modules/sfdmu-run/custom-addons/package/ISfdmuRunCustomAddonScript.ts
@@ -25,6 +25,7 @@ export default interface ISfdmuRunCustomAddonScript {
     pollingQueryTimeoutMs: number;
     concurrencyMode: "Serial" | "Parallel";
     bulkThreshold: number;
+    queryBulkApiThreshold: number;
     bulkApiVersion: string;
     bulkApiV1BatchSize: number;
     restApiBatchSize: number;

diff --git a/src/modules/components/common_components/logger.ts b/src/modules/components/common_components/logger.ts
@@ -190,6 +190,8 @@ export enum RESOURCES {
   mappingTargetRecords = "mappingTargetRecords",
   retrievingData = "retrievingData",
   retrievingDataCompleted = "retrievingDataCompleted",
+  usingRestApi = "usingRestApi",
+  usingBulkAPIQuery = "usingBulkAPIQuery",
   queryingAll = "queryingAll",
   queryingIn = "queryingIn",
   queryingIn2 = "queryingIn2",

diff --git a/src/modules/components/common_components/sfdx.ts b/src/modules/components/common_components/sfdx.ts
@@ -71,7 +71,7 @@ export class Sfdx implements IAppSfdxService, IFieldMapping {
    *  Handles composite fields.
    *
    * @param {string} soql The SOQL query
-   * @param {boolean} useBulkQueryApi true to use Bulk Query Api instead of the Collection Api
+   * @param {boolean} useBulkQueryApi true to use Bulk Query Api instead of the Rest Api
    * @param {boolean} bulkQueryPollTimeout If set and useBuilkQueryApi is true, polling for the query results will timeout after this amount of milliseconds
    * @returns {Promise<QueryResult<object>>}
    * @memberof ApiSf
@@ -199,6 +199,13 @@ export class Sfdx implements IAppSfdxService, IFieldMapping {
       let parsedQuery = parseQuery(soql);
       soql = this.sourceQueryToTarget(soql, parsedQuery.sObject).query;
 
+      if (!useBulkQueryApi) {
+        this.logger.infoVerbose(RESOURCES.usingRestApi, parsedQuery.sObject);
+      } else {
+        this.logger.infoVerbose(RESOURCES.usingBulkAPIQuery, parsedQuery.sObject);
+      }
+
+
       // Query records /////
       let records = [].concat(await ___queryAsync(soql));
       if (/FROM Group([\s]+|$)/i.test(soql)) {

diff --git a/src/modules/models/common_models/helpers_classes.ts b/src/modules/models/common_models/helpers_classes.ts
@@ -341,8 +341,10 @@ export class TaskOrgData {
      * @memberof TaskOrgData
      */
     get useBulkQueryApi(): boolean {
-        return this.isSource ? this.task.sourceTotalRecorsCount > CONSTANTS.QUERY_BULK_API_THRESHOLD :
-            this.task.targetTotalRecorsCount > CONSTANTS.QUERY_BULK_API_THRESHOLD;
+        const bulkThreshold = this.task.script.queryBulkApiThreshold || CONSTANTS.QUERY_BULK_API_THRESHOLD;
+        return this.isSource ? this.task.sourceTotalRecorsCount >= bulkThreshold :
+            this.task.targetTotalRecorsCount >= bulkThreshold;
+
     }
 
     /**

diff --git a/src/modules/models/script_models/script.ts b/src/modules/models/script_models/script.ts
@@ -78,6 +78,7 @@ export default class Script implements IAppScript, ISfdmuRunScript {
   pollingQueryTimeoutMs: number = CONSTANTS.DEFAULT_POLLING_QUERY_TIMEOUT_MS;
   concurrencyMode: "Serial" | "Parallel" = "Parallel";
   bulkThreshold: number = CONSTANTS.DEFAULT_BULK_API_THRESHOLD_RECORDS;
+  queryBulkApiThreshold: number = CONSTANTS.QUERY_BULK_API_THRESHOLD;
   bulkApiVersion: string = CONSTANTS.DEFAULT_BULK_API_VERSION;
   bulkApiV1BatchSize: number;
   restApiBatchSize: number;