Adds batching to the Iterable Create or Update User destination (#2343)

* Adds a performBatch method to the Iterable updateUser destination.

Refactors some of the payload transformation code from the perform method, to be reused in the performBatch.

* Adds enable_batching and batch_size to the action definition field list. (Disabled and 500 by default, respectively, with only the former being visible/configurable.)

Re-generates types.

* Re-generates Jest snapshots following changes for Iterable updateUser batch support

packages/destination-actions/src/destinations/iterable/updateUser/__tests__/__snapshots__/snapshot.test.ts.snap

@@ -2,11 +2,13 @@
 
 exports[`Testing snapshot for Iterable's updateUser destination action: all fields 1`] = `
 Object {
+  "batch_size": -19733782860922.88,
   "dataFields": Object {
     "phoneNumber": "^qMtTUbd*oM",
     "testType": "^qMtTUbd*oM",
   },
   "email": "[email protected]",
+  "enable_batching": true,
   "mergeNestedObjects": true,
   "userId": "^qMtTUbd*oM",
 }

packages/destination-actions/src/destinations/iterable/updateUser/__tests__/index.test.ts

@@ -5,115 +5,175 @@ import Destination from '../../index'
 const testDestination = createTestIntegration(Destination)
 
 describe('Iterable.updateUser', () => {
-  it('works with default mappings', async () => {
-    const event = createTestEvent({ type: 'identify' })
+  describe('perform', () => {
+    it('works with default mappings', async () => {
+      const event = createTestEvent({ type: 'identify' })
 
-    nock('https://api.iterable.com/api').post('/users/update').reply(200, {})
+      nock('https://api.iterable.com/api').post('/users/update').reply(200, {})
 
-    const responses = await testDestination.testAction('updateUser', {
-      event,
-      useDefaultMappings: true
+      const responses = await testDestination.testAction('updateUser', {
+        event,
+        useDefaultMappings: true
+      })
+
+      expect(responses.length).toBe(1)
+      expect(responses[0].status).toBe(200)
     })
 
-    expect(responses.length).toBe(1)
-    expect(responses[0].status).toBe(200)
-  })
+    it('throws an error if `email` or `userId` are not defined', async () => {
+      const event = createTestEvent({
+        type: 'identify',
+        traits: {
+          firstName: 'Johnny',
+          lastName: 'Depp'
+        }
+      })
 
-  it('throws an error if `email` or `userId` are not defined', async () => {
-    const event = createTestEvent({
-      type: 'identify',
-      traits: {
-        firstName: 'Johnny',
-        lastName: 'Depp'
-      }
+      await expect(
+        testDestination.testAction('updateUser', {
+          event
+        })
+      ).rejects.toThrowError(PayloadValidationError)
     })
 
-    await expect(
-      testDestination.testAction('updateUser', {
-        event
+    it('maps phoneNumber in dataFields', async () => {
+      const event = createTestEvent({
+        type: 'identify',
+        userId: 'user1234',
+        traits: {
+          phone: '+14158675309'
+        }
       })
-    ).rejects.toThrowError(PayloadValidationError)
-  })
 
-  it('maps phoneNumber in dataFields', async () => {
-    const event = createTestEvent({
-      type: 'identify',
-      userId: 'user1234',
-      traits: {
-        phone: '+14158675309'
-      }
-    })
+      nock('https://api.iterable.com/api').post('/users/update').reply(200, {})
 
-    nock('https://api.iterable.com/api').post('/users/update').reply(200, {})
+      const responses = await testDestination.testAction('updateUser', {
+        event,
+        useDefaultMappings: true
+      })
 
-    const responses = await testDestination.testAction('updateUser', {
-      event,
-      useDefaultMappings: true
+      expect(responses.length).toBe(1)
+      expect(responses[0].options.json).toMatchObject({
+        userId: 'user1234',
+        dataFields: {
+          phoneNumber: '+14158675309'
+        }
+      })
     })
 
-    expect(responses.length).toBe(1)
-    expect(responses[0].options.json).toMatchObject({
-      userId: 'user1234',
-      dataFields: {
-        phoneNumber: '+14158675309'
-      }
-    })
-  })
+    it('should success with mapping of preset and `identify` call', async () => {
+      const event = createTestEvent({
+        type: 'identify',
+        traits: {
+          phone: '+14158675309'
+        }
+      })
 
-  it('should success with mapping of preset and `identify` call', async () => {
-    const event = createTestEvent({
-      type: 'identify',
-      traits: {
-        phone: '+14158675309'
-      }
+      nock('https://api.iterable.com/api').post('/users/update').reply(200, {})
+
+      const responses = await testDestination.testAction('updateUser', {
+        event,
+        // Using the mapping of presets with event type 'track'
+        mapping: {
+          dataFields: {
+            '@path': '$.traits'
+          }
+        },
+        useDefaultMappings: true
+      })
+
+      expect(responses.length).toBe(1)
+      expect(responses[0].status).toBe(200)
     })
 
-    nock('https://api.iterable.com/api').post('/users/update').reply(200, {})
+    it('should allow passing null values for phoneNumber', async () => {
+      const event = createTestEvent({
+        type: 'identify',
+        userId: 'user1234',
+        traits: {
+          phone: null,
+          trait1: null
+        }
+      })
+
+      nock('https://api.iterable.com/api').post('/users/update').reply(200, {})
+
+      const responses = await testDestination.testAction('updateUser', {
+        event,
+        mapping: {
+          dataFields: {
+            '@path': '$.traits'
+          }
+        },
+        useDefaultMappings: true
+      })
 
-    const responses = await testDestination.testAction('updateUser', {
-      event,
-      // Using the mapping of presets with event type 'track'
-      mapping: {
+      expect(responses.length).toBe(1)
+      expect(responses[0].options.json).toMatchObject({
+        userId: 'user1234',
         dataFields: {
-          '@path': '$.traits'
+          phoneNumber: null,
+          trait1: null
         }
-      },
-      useDefaultMappings: true
+      })
     })
-
-    expect(responses.length).toBe(1)
-    expect(responses[0].status).toBe(200)
   })
+  describe('performBatch', () => {
+    it('works with default mappings', async () => {
+      const events = [createTestEvent({ type: 'identify' }), createTestEvent({ type: 'identify' })]
 
-  it('should allow passing null values for phoneNumber', async () => {
-    const event = createTestEvent({
-      type: 'identify',
-      userId: 'user1234',
-      traits: {
-        phone: null,
-        trait1: null
-      }
-    })
+      nock('https://api.iterable.com/api').post('/users/bulkUpdate').reply(200, {})
 
-    nock('https://api.iterable.com/api').post('/users/update').reply(200, {})
+      const responses = await testDestination.testBatchAction('updateUser', {
+        events,
+        useDefaultMappings: true
+      })
 
-    const responses = await testDestination.testAction('updateUser', {
-      event,
-      mapping: {
-        dataFields: {
-          '@path': '$.traits'
-        }
-      },
-      useDefaultMappings: true
+      expect(responses.length).toBe(1)
+      expect(responses[0].status).toBe(200)
     })
+    it('maps phoneNumber in dataFields', async () => {
+      const events = [
+        createTestEvent({
+          type: 'identify',
+          userId: 'user1234',
+          traits: {
+            phone: '+14158675309'
+          }
+        }),
+        createTestEvent({
+          type: 'identify',
+          userId: 'user5678',
+          traits: {
+            phone: '+24158675309'
+          }
+        })
+      ]
+
+      nock('https://api.iterable.com/api').post('/users/bulkUpdate').reply(200, {})
+
+      const responses = await testDestination.testBatchAction('updateUser', {
+        events,
+        useDefaultMappings: true
+      })
 
-    expect(responses.length).toBe(1)
-    expect(responses[0].options.json).toMatchObject({
-      userId: 'user1234',
-      dataFields: {
-        phoneNumber: null,
-        trait1: null
-      }
+      expect(responses.length).toBe(1)
+      expect(responses[0].options.json).toMatchObject({
+        users: [
+          {
+            userId: 'user1234',
+            dataFields: {
+              phoneNumber: '+14158675309'
+            }
+          },
+          {
+            userId: 'user5678',
+            dataFields: {
+              phoneNumber: '+24158675309'
+            }
+          }
+        ]
+      })
     })
   })
 })

packages/destination-actions/src/destinations/iterable/updateUser/index.ts

@@ -11,6 +11,35 @@ import {
 } from '../shared-fields'
 import { convertDatesInObject, getRegionalEndpoint } from '../utils'
 
+interface UserUpdateRequestPayload {
+  email?: string
+  userId?: string
+  dataFields?: {
+    [k: string]: unknown
+  }
+  mergeNestedObjects?: boolean
+}
+
+interface BulkUserUpdateRequestPayload {
+  users: UserUpdateRequestPayload[]
+}
+
+const transformIterableUserPayload: (payload: Payload) => UserUpdateRequestPayload = (payload) => {
+  // Store the phoneNumber value before deleting from the top-level object
+  const phoneNumber = payload.phoneNumber
+  delete payload.phoneNumber
+
+  const formattedDataFields = convertDatesInObject(payload.dataFields ?? {})
+  const userUpdateRequest: UserUpdateRequestPayload = {
+    ...payload,
+    dataFields: {
+      ...formattedDataFields,
+      phoneNumber: phoneNumber
+    }
+  }
+  return userUpdateRequest
+}
+
 const action: ActionDefinition<Settings, Payload> = {
   title: 'Upsert User',
   description: 'Creates or updates a user',
@@ -31,41 +60,46 @@ const action: ActionDefinition<Settings, Payload> = {
     },
     mergeNestedObjects: {
       ...MERGE_NESTED_OBJECTS_FIELD
+    },
+    enable_batching: {
+      label: 'Enable Batching',
+      description: 'When enabled, Segment will send data to Iterable in batches of up to 500',
+      type: 'boolean',
+      required: false,
+      default: false
+    },
+    batch_size: {
+      label: 'Batch Size',
+      description: 'Maximum number of events to include in each batch. Actual batch sizes may be lower.',
+      type: 'number',
+      unsafe_hidden: true,
+      required: false,
+      default: 500
     }
   },
   perform: (request, { payload, settings }) => {
-    const { email, userId, dataFields } = payload
-
-    if (!email && !userId) {
+    if (!payload.email && !payload.userId) {
       throw new PayloadValidationError('Must include email or userId.')
     }
 
-    interface UserUpdateRequest {
-      email?: string
-      userId?: string
-      dataFields?: {
-        [k: string]: unknown
-      }
-      mergeNestedObjects?: boolean
-    }
-
-    // Store the phoneNumber value before deleting from the top-level object
-    const phoneNumber = payload.phoneNumber
-    delete payload.phoneNumber
+    const updateUserRequestPayload: UserUpdateRequestPayload = transformIterableUserPayload(payload)
 
-    const formattedDataFields = convertDatesInObject(dataFields ?? {})
-    const userUpdateRequest: UserUpdateRequest = {
-      ...payload,
-      dataFields: {
-        ...formattedDataFields,
-        phoneNumber: phoneNumber
-      }
+    const endpoint = getRegionalEndpoint('updateUser', settings.dataCenterLocation as DataCenterLocation)
+    return request(endpoint, {
+      method: 'post',
+      json: updateUserRequestPayload,
+      timeout: Math.max(30_000, DEFAULT_REQUEST_TIMEOUT)
+    })
+  },
+  performBatch: (request, { settings, payload }) => {
+    const bulkUpdateUserRequestPayload: BulkUserUpdateRequestPayload = {
+      users: payload.map(transformIterableUserPayload)
     }
 
-    const endpoint = getRegionalEndpoint('updateUser', settings.dataCenterLocation as DataCenterLocation)
+    const endpoint = getRegionalEndpoint('bulkUpdateUser', settings.dataCenterLocation as DataCenterLocation)
     return request(endpoint, {
       method: 'post',
-      json: userUpdateRequest,
+      json: bulkUpdateUserRequestPayload,
       timeout: Math.max(30_000, DEFAULT_REQUEST_TIMEOUT)
     })
   }