From 4cf4deb554275087a1f2df3d968bfcfce9969605 Mon Sep 17 00:00:00 2001 From: Shwetabh Kumar Date: Wed, 15 Nov 2023 19:47:30 +0530 Subject: [PATCH 1/4] feat: Optimize Contacts Sync --- apps/xero/utils.py | 93 ++++++++++++++++++++-------------------------- requirements.txt | 2 +- 2 files changed, 42 insertions(+), 53 deletions(-) diff --git a/apps/xero/utils.py b/apps/xero/utils.py index f9b5eb1b..fb010861 100644 --- a/apps/xero/utils.py +++ b/apps/xero/utils.py @@ -68,12 +68,20 @@ def get_suppliers(self): tenant_mapping = TenantMapping.objects.get(workspace_id=self.workspace_id) self.connection.set_tenant_id(tenant_mapping.tenant_id) - suppliers_generator = self.connection.contacts.list_all_generator() + params = { + 'where': 'IsSupplier=true', + 'includeArchived': 'false', + } + + suppliers_generator = self.connection.contacts.list_all_generator(**params) merchant_names: List[str] = [] + for suppliers in suppliers_generator: for supplier in suppliers["Contacts"]: - if supplier["IsSupplier"] and supplier["ContactStatus"] == "ACTIVE": - merchant_names.append(supplier["Name"]) + merchant_names.append(supplier["Name"]) + + sleep(1) + return merchant_names def get_or_create_contact( @@ -268,6 +276,11 @@ def sync_contacts(self): updated_at = get_last_synced_at(self.workspace_id, "CONTACT") + params = { + 'where': 'IsCustomer=false', + 'includeArchived': 'true' + } + contacts_generator = self.connection.contacts.list_all_generator( modified_after=updated_at ) @@ -288,6 +301,7 @@ def sync_contacts(self): "value": contact["Name"], "destination_id": contact["ContactID"], "detail": detail, + "active": True if contact["ContactStatus"] == "ACTIVE" else False, } ) @@ -305,61 +319,36 @@ def sync_customers(self): self.connection.set_tenant_id(tenant_mapping.tenant_id) - customers_generator = self.connection.contacts.list_all_generator() - - customer_attributes = [] - - destination_attributes = DestinationAttribute.objects.filter( - workspace_id=self.workspace_id, - attribute_type="CUSTOMER", - display_name="Customer", - ).values("destination_id", "value", "detail") - - disabled_fields_map = {} + params = { + 'where': 'IsCustomer=true', + 'includeArchived': 'true', + } - for destination_attribute in destination_attributes: - disabled_fields_map[destination_attribute["destination_id"]] = { - "value": destination_attribute["value"], - "detail": destination_attribute["detail"], - } + customers_generator = self.connection.contacts.list_all_generator(**params) for customers in customers_generator: - for customer in customers["Contacts"]: - if customer["IsCustomer"]: - customer_attributes.append( - { - "attribute_type": "CUSTOMER", - "display_name": "Customer", - "value": customer["Name"], - "destination_id": customer["ContactID"], - "detail": { - "email": customer["EmailAddress"] - if "EmailAddress" in customer - else None - }, - "active": True if customer["ContactStatus"] == "ACTIVE" else False, - } - ) + customer_attributes = [] - if (customer["ContactStatus"] == "ACTIVE" and customer["ContactID"] in disabled_fields_map): - disabled_fields_map.pop(customer["ContactID"]) - sleep(0.5) + for customer in customers["Contacts"]: + customer_attributes.append( + { + "attribute_type": "CUSTOMER", + "display_name": "Customer", + "value": customer["Name"], + "destination_id": customer["ContactID"], + "detail": { + "email": customer["EmailAddress"] + if "EmailAddress" in customer + else None + }, + "active": True if customer["ContactStatus"] == "ACTIVE" else False, + } + ) - for destination_id in disabled_fields_map: - customer_attributes.append( - { - "attribute_type": "CUSTOMER", - "display_name": "customer", - "value": disabled_fields_map[destination_id]["value"], - "destination_id": destination_id, - "active": False, - "detail": disabled_fields_map[destination_id]["detail"], - } + DestinationAttribute.bulk_create_or_update_destination_attributes( + customer_attributes, "CUSTOMER", self.workspace_id, True ) - - DestinationAttribute.bulk_create_or_update_destination_attributes( - customer_attributes, "CUSTOMER", self.workspace_id, True - ) + sleep(1) return [] diff --git a/requirements.txt b/requirements.txt index 81de6055..60dd97cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,7 +41,7 @@ Unidecode==1.1.2 urllib3==1.26.11 wcwidth==0.1.8 wrapt==1.12.1 -xerosdk==0.13.0 +xerosdk==0.14.0 pytest==7.1.2 pytest-cov==3.0.0 pytest-django==4.5.2 From 40a91fe1fca21d8d45409c555dfbe584aa2bb5fb Mon Sep 17 00:00:00 2001 From: Shwetabh Kumar Date: Thu, 16 Nov 2023 09:09:24 +0530 Subject: [PATCH 2/4] Adding sleep for sync_contacts too --- apps/xero/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/xero/utils.py b/apps/xero/utils.py index fb010861..97828851 100644 --- a/apps/xero/utils.py +++ b/apps/xero/utils.py @@ -282,7 +282,7 @@ def sync_contacts(self): } contacts_generator = self.connection.contacts.list_all_generator( - modified_after=updated_at + modified_after=updated_at, **params ) for contacts in contacts_generator: @@ -308,6 +308,7 @@ def sync_contacts(self): DestinationAttribute.bulk_create_or_update_destination_attributes( contact_attributes, "CONTACT", self.workspace_id, True ) + sleep(1) return [] From 1dee1903c34c2f4ae4447bb7f164a99394d9a4f7 Mon Sep 17 00:00:00 2001 From: Shwetabh Kumar Date: Thu, 16 Nov 2023 14:34:15 +0530 Subject: [PATCH 3/4] Fixing test --- tests/test_xero/test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_xero/test_utils.py b/tests/test_xero/test_utils.py index 414d0f04..f0064dc6 100644 --- a/tests/test_xero/test_utils.py +++ b/tests/test_xero/test_utils.py @@ -184,14 +184,14 @@ def test_sync_customers(mocker, db): customers_count = DestinationAttribute.objects.filter( workspace_id=workspace_id, attribute_type="CUSTOMER" ).count() - assert customers_count == 14 + assert customers_count == 62 xero_connection.sync_customers() new_customers_count = DestinationAttribute.objects.filter( workspace_id=workspace_id, attribute_type="CUSTOMER" ).count() - assert new_customers_count == 14 + assert new_customers_count == 62 def test_sync_tracking_categories(mocker, db): From 026351917dbe57c051bc8a9113542831071c6aa3 Mon Sep 17 00:00:00 2001 From: Shwetabh Kumar Date: Thu, 16 Nov 2023 14:39:41 +0530 Subject: [PATCH 4/4] Fixing test --- tests/test_xero/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_xero/test_utils.py b/tests/test_xero/test_utils.py index f0064dc6..a64adca1 100644 --- a/tests/test_xero/test_utils.py +++ b/tests/test_xero/test_utils.py @@ -184,7 +184,7 @@ def test_sync_customers(mocker, db): customers_count = DestinationAttribute.objects.filter( workspace_id=workspace_id, attribute_type="CUSTOMER" ).count() - assert customers_count == 62 + assert customers_count == 14 xero_connection.sync_customers()