feat: Add attribute limits (#678)

* feat: Add limits to attributes

* bump up version

* added test

* pylint fix

* test fixed

apps/quickbooks_online/utils.py

@@ -1,6 +1,7 @@
 import json
 import logging
 from datetime import datetime, timedelta
+from django.utils import timezone
 from typing import Dict, List
 
 import unidecode
@@ -32,7 +33,15 @@
 logger = logging.getLogger(__name__)
 logger.level = logging.INFO
 
-SYNC_UPPER_LIMIT = {'customers': 25000}
+SYNC_UPPER_LIMIT = {
+    'customers': 30000,
+    'classes': 5000,
+    'accounts': 3000,
+    'departments': 2000,
+    'vendors': 20000,
+    'tax_codes': 200,
+    'items': 3000
+}
 
 
 def format_special_characters(value: str) -> str:
@@ -150,10 +159,31 @@ def get_tax_inclusive_amount(self, amount, default_tax_code_id):
 
         return tax_inclusive_amount
 
+    def is_sync_allowed(self, attribute_type: str, attribute_count: int):
+        """
+        Checks if the sync is allowed
+
+        Returns:
+            bool: True
+        """
+        if attribute_count > SYNC_UPPER_LIMIT[attribute_type]:
+            workspace_created_at = Workspace.objects.get(id=self.workspace_id).created_at
+            if workspace_created_at > timezone.make_aware(datetime(2024, 10, 1), timezone.get_current_timezone()):
+                return False
+            else:
+                return True
+
+        return True
+
     def sync_items(self):
         """
         Get items
         """
+        attribute_count = self.connection.items.count()
+        if not self.is_sync_allowed(attribute_type = 'items', attribute_count = attribute_count):
+            logger.info('Skipping sync of items for workspace %s as it has %s counts which is over the limit', self.workspace_id, attribute_count)
+            return
+
         items_generator = self.connection.items.get_all_generator()
         general_settings = WorkspaceGeneralSettings.objects.filter(workspace_id=self.workspace_id).first()
 
@@ -183,6 +213,11 @@ def sync_accounts(self):
         """
         Get accounts
         """
+        attribute_count = self.connection.accounts.count()
+        if not self.is_sync_allowed(attribute_type = 'accounts', attribute_count=attribute_count):
+            logger.info('Skipping sync of accounts for workspace %s as it has %s counts which is over the limit', self.workspace_id, attribute_count)
+            return
+
         accounts_generator = self.connection.accounts.get_all_generator()
         category_sync_version = 'v2'
         general_settings = WorkspaceGeneralSettings.objects.filter(workspace_id=self.workspace_id).first()
@@ -302,6 +337,11 @@ def sync_departments(self):
         """
         Get departments
         """
+        attribute_count = self.connection.departments.count()
+        if not self.is_sync_allowed(attribute_type = 'departments', attribute_count = attribute_count):
+            logger.info('Skipping sync of department for workspace %s as it has %s counts which is over the limit', self.workspace_id, attribute_count)
+            return
+
         departments_generator = self.connection.departments.get_all_generator()
 
         for departments in departments_generator:
@@ -317,6 +357,11 @@ def sync_tax_codes(self):
         """
         Get Tax Codes
         """
+        attribute_count = self.connection.tax_codes.count()
+        if not self.is_sync_allowed(attribute_type = 'tax_codes', attribute_count = attribute_count):
+            logger.info('Skipping sync of tax_codes for workspace %s as it has %s counts which is over the limit', self.workspace_id, attribute_count)
+            return
+
         tax_codes_generator = self.connection.tax_codes.get_all_generator()
 
         for tax_codes in tax_codes_generator:
@@ -345,6 +390,11 @@ def sync_vendors(self):
         """
         Get vendors
         """
+        attribute_count = self.connection.vendors.count()
+        if not self.is_sync_allowed(attribute_type = 'vendors', attribute_count = attribute_count):
+            logger.info('Skipping sync of vendors for workspace %s as it has %s counts which is over the limit', self.workspace_id, attribute_count)
+            return
+
         vendors_generator = self.connection.vendors.get_all_generator()
 
         for vendors in vendors_generator:
@@ -429,6 +479,11 @@ def sync_classes(self):
         """
         Get classes
         """
+        attribute_count = self.connection.classes.count()
+        if not self.is_sync_allowed(attribute_type = 'classes', attribute_count = attribute_count):
+            logger.info('Skipping sync of classes for workspace %s as it has %s counts which is over the limit', self.workspace_id, attribute_count)
+            return
+
         classes_generator = self.connection.classes.get_all_generator()
 
         for classes in classes_generator:
@@ -444,31 +499,34 @@ def sync_customers(self):
         """
         Get customers
         """
-        customers_count = self.connection.customers.count()
-        if customers_count < SYNC_UPPER_LIMIT['customers']:
-            customers_generator = self.connection.customers.get_all_generator()
+        attribute_count = self.connection.customers.count()
+        if not self.is_sync_allowed(attribute_type = 'customers', attribute_count = attribute_count):
+            logger.info('Skipping sync of customers for workspace %s as it has %s counts which is over the limit', self.workspace_id, attribute_count)
+            return
+
+        customers_generator = self.connection.customers.get_all_generator()
 
-            for customers in customers_generator:
-                customer_attributes = []
-                for customer in customers:
-                    value = format_special_characters(customer['FullyQualifiedName'])
-                    if customer['Active'] and value:
-                        customer_attributes.append({'attribute_type': 'CUSTOMER', 'display_name': 'customer', 'value': value, 'destination_id': customer['Id'], 'active': True})
+        for customers in customers_generator:
+            customer_attributes = []
+            for customer in customers:
+                value = format_special_characters(customer['FullyQualifiedName'])
+                if customer['Active'] and value:
+                    customer_attributes.append({'attribute_type': 'CUSTOMER', 'display_name': 'customer', 'value': value, 'destination_id': customer['Id'], 'active': True})
 
-                DestinationAttribute.bulk_create_or_update_destination_attributes(customer_attributes, 'CUSTOMER', self.workspace_id, True)
+            DestinationAttribute.bulk_create_or_update_destination_attributes(customer_attributes, 'CUSTOMER', self.workspace_id, True)
 
-            last_synced_time = get_last_synced_time(self.workspace_id, 'PROJECT')
+        last_synced_time = get_last_synced_time(self.workspace_id, 'PROJECT')
 
-            # get the inactive customers generator
-            inactive_customers_generator = self.connection.customers.get_inactive(last_synced_time)
+        # get the inactive customers generator
+        inactive_customers_generator = self.connection.customers.get_inactive(last_synced_time)
 
-            for inactive_customers in inactive_customers_generator:
-                inactive_customer_attributes = []
-                for inactive_customer in inactive_customers:
-                    display_name = inactive_customer['DisplayName'].replace(" (deleted)", "").rstrip()
-                    inactive_customer_attributes.append({'attribute_type': 'CUSTOMER', 'display_name': 'customer', 'value': display_name, 'destination_id': inactive_customer['Id'], 'active': False})
+        for inactive_customers in inactive_customers_generator:
+            inactive_customer_attributes = []
+            for inactive_customer in inactive_customers:
+                display_name = inactive_customer['DisplayName'].replace(" (deleted)", "").rstrip()
+                inactive_customer_attributes.append({'attribute_type': 'CUSTOMER', 'display_name': 'customer', 'value': display_name, 'destination_id': inactive_customer['Id'], 'active': False})
 
-                DestinationAttribute.bulk_create_or_update_destination_attributes(inactive_customer_attributes, 'CUSTOMER', self.workspace_id, True)
+            DestinationAttribute.bulk_create_or_update_destination_attributes(inactive_customer_attributes, 'CUSTOMER', self.workspace_id, True)
 
         return []
 

requirements.txt

@@ -35,7 +35,7 @@ psycopg2-binary==2.8.4
 pylint==2.7.4
 python-dateutil==2.8.1
 pytz==2019.3
-qbosdk==0.17.3
+qbosdk==0.18.0
 redis==3.3.11
 requests==2.25.0
 sentry-sdk==1.19.1

tests/test_fyle_integrations_imports/test_modules/test_base.py

@@ -223,8 +223,13 @@ def test_auto_create_destination_attributes(mocker, db):
             'fyle_integrations_platform_connector.apis.Categories.post_bulk',
             return_value=[]
         )
+        mocker.patch('qbosdk.apis.Items.count',return_value=0)
         mocker.patch('qbosdk.apis.Items.get_all_generator',return_value=[])
         mocker.patch('qbosdk.apis.Items.get_inactive',return_value=[])
+        mocker.patch(
+            'qbosdk.apis.Accounts.count',
+            return_value=0
+        )
         mocker.patch(
             'qbosdk.apis.Accounts.get_all_generator',
             return_value=[]

tests/test_fyle_integrations_imports/test_modules/test_categories.py

@@ -16,6 +16,7 @@ def test_sync_destination_attributes(mocker, db):
     workspace_id = 2
 
     # import categories
+    mocker.patch('qbosdk.apis.Accounts.count', return_value=10)
     mocker.patch('qbosdk.apis.Accounts.get_all_generator', return_value=[categories_data['create_new_auto_create_categories_destination_attributes_accounts']])
     mocker.patch('qbosdk.apis.Accounts.get_inactive', return_value=[])
     mocker.patch('qbosdk.apis.Items.get_inactive', return_value=[])
@@ -35,6 +36,7 @@ def test_sync_destination_attributes(mocker, db):
     # import items
     category = Category(2, 'ACCOUNT', None,  qbo_connection, ['items'], True, False, ['Expense', 'Fixed Asset'])
     WorkspaceGeneralSettings.objects.filter(workspace_id=workspace_id).update(import_categories=True, import_items=True)
+    mocker.patch('qbosdk.apis.Items.count', return_value=10)
     mocker.patch('qbosdk.apis.Items.get_all_generator', return_value=[categories_data['create_new_auto_create_categories_destination_attributes_items']])
 
     destination_attributes_count = DestinationAttribute.objects.filter(workspace_id=workspace_id, attribute_type='ACCOUNT', display_name='Item').count()
@@ -48,6 +50,7 @@ def test_sync_destination_attributes(mocker, db):
     # import items + import categories
     category = Category(2, 'ACCOUNT', None,  qbo_connection, ['items', 'accounts'], True, False, ['Expense', 'Fixed Asset'])
     WorkspaceGeneralSettings.objects.filter(workspace_id=workspace_id).update(import_categories=True, import_items=True)
+    mocker.patch('qbosdk.apis.Accounts.count', return_value=10)
     mocker.patch('qbosdk.apis.Accounts.get_all_generator', return_value=[categories_data['create_new_auto_create_categories_destination_attributes_accounts_sync']])
     mocker.patch('qbosdk.apis.Items.get_all_generator', return_value=[categories_data['create_new_auto_create_categories_destination_attributes_items_sync']])
 
@@ -115,6 +118,14 @@ def test_auto_create_destination_attributes(mocker, db):
     DestinationAttribute.objects.filter(workspace_id=workspace_id, attribute_type='ACCOUNT').delete()
     ExpenseAttribute.objects.filter(workspace_id=workspace_id, attribute_type='CATEGORY').delete()
 
+    mocker.patch(
+        'qbosdk.apis.Accounts.count',
+        return_value=0
+    )
+    mocker.patch(
+        'qbosdk.apis.Items.count',
+        return_value=0
+    )
     mocker.patch(
         'qbosdk.apis.Accounts.get_inactive',
         return_value=[]
@@ -138,7 +149,12 @@ def test_auto_create_destination_attributes(mocker, db):
             'fyle_integrations_platform_connector.apis.Categories.post_bulk',
             return_value=[]
         )
+        mocker.patch('qbosdk.apis.Items.count',return_value=0)
         mocker.patch('qbosdk.apis.Items.get_all_generator',return_value=[])
+        mocker.patch(
+            'qbosdk.apis.Accounts.count',
+            return_value=10
+        )
         mocker.patch(
             'qbosdk.apis.Accounts.get_all_generator',
             return_value=[categories_data['create_new_auto_create_categories_destination_attributes_accounts']]
@@ -181,7 +197,12 @@ def test_auto_create_destination_attributes(mocker, db):
             'fyle_integrations_platform_connector.apis.Categories.post_bulk',
             return_value=[]
         )
+        mocker.patch('qbosdk.apis.Items.count',return_value=0)
         mocker.patch('qbosdk.apis.Items.get_all_generator',return_value=[])
+        mocker.patch(
+            'qbosdk.apis.Accounts.count',
+            return_value=10
+        )
         mocker.patch(
             'qbosdk.apis.Accounts.get_inactive',
             return_value=[categories_data['create_new_auto_create_categories_destination_attributes_accounts_disable_case']]
@@ -228,7 +249,12 @@ def test_auto_create_destination_attributes(mocker, db):
             'fyle_integrations_platform_connector.apis.Categories.post_bulk',
             return_value=[]
         )
+        mocker.patch('qbosdk.apis.Items.count',return_value=0)
         mocker.patch('qbosdk.apis.Items.get_all_generator',return_value=[])
+        mocker.patch(
+            'qbosdk.apis.Accounts.count',
+            return_value=10
+        )
         mocker.patch(
             'qbosdk.apis.Accounts.get_inactive',
             return_value=[categories_data['create_new_auto_create_categories_destination_attributes_accounts']]
@@ -310,10 +336,18 @@ def test_auto_create_destination_attributes(mocker, db):
             'fyle_integrations_platform_connector.apis.Categories.post_bulk',
             return_value=[]
         )
+        mocker.patch(
+            'qbosdk.apis.Items.count',
+            return_value=10
+        )
         mocker.patch(
             'qbosdk.apis.Items.get_inactive',
             return_value=[categories_data['create_new_auto_create_categories_destination_attributes_items_disabled_case']]
         )
+        mocker.patch(
+            'qbosdk.apis.Accounts.count',
+            return_value=0
+        )
         mocker.patch(
             'qbosdk.apis.Accounts.get_all_generator',
             return_value=[]
@@ -360,10 +394,18 @@ def test_auto_create_destination_attributes(mocker, db):
             'fyle_integrations_platform_connector.apis.Categories.post_bulk',
             return_value=[]
         )
+        mocker.patch(
+            'qbosdk.apis.Items.count',
+            return_value=10
+        )
         mocker.patch(
             'qbosdk.apis.Items.get_inactive',
             return_value=[categories_data['create_new_auto_create_categories_destination_attributes_items']]
         )
+        mocker.patch(
+            'qbosdk.apis.Accounts.count',
+            return_value=0
+        )
         mocker.patch(
             'qbosdk.apis.Accounts.get_all_generator',
             return_value=[]
@@ -398,10 +440,18 @@ def test_auto_create_destination_attributes(mocker, db):
             'fyle_integrations_platform_connector.apis.Categories.post_bulk',
             return_value=[]
         )
+        mocker.patch(
+            'qbosdk.apis.Items.count',
+            return_value=10
+        )
         mocker.patch(
             'qbosdk.apis.Items.get_inactive',
             return_value=[categories_data['create_new_auto_create_categories_destination_attributes_items']]
         )
+        mocker.patch(
+            'qbosdk.apis.Accounts.count',
+            return_value=0
+        )
         mocker.patch(
             'qbosdk.apis.Accounts.get_all_generator',
             return_value=[]

tests/test_fyle_integrations_imports/test_modules/test_cost_center.py

@@ -14,6 +14,10 @@
 def test_sync_destination_attributes(mocker, db):
     workspace_id = 3
 
+    mocker.patch(
+        'qbosdk.apis.Classes.count',
+        return_value=10
+    )
     mocker.patch(
         'qbosdk.apis.Classes.get_all_generator',
         return_value=[cost_center_data['create_new_auto_create_cost_centers_destination_attributes']]
@@ -87,6 +91,10 @@ def test_auto_create_destination_attributes(mocker, db):
             'fyle_integrations_platform_connector.apis.CostCenters.post_bulk',
             return_value=[]
         )
+        mocker.patch(
+            'qbosdk.apis.Classes.count',
+            return_value=10
+        )
         mocker.patch(
             'qbosdk.apis.Classes.get_all_generator',
             return_value=[cost_center_data['create_new_auto_create_cost_centers_destination_attributes']]
@@ -120,6 +128,10 @@ def test_auto_create_destination_attributes(mocker, db):
             'fyle_integrations_platform_connector.apis.CostCenters.post_bulk',
             return_value=[]
         )
+        mocker.patch(
+            'qbosdk.apis.Classes.count',
+            return_value=10
+        )
         mocker.patch(
             'qbosdk.apis.Classes.get_all_generator',
             return_value=[cost_center_data['create_new_auto_create_cost_centers_destination_attributes_subsequent_run']]

tests/test_fyle_integrations_imports/test_modules/test_expense_custom_fields.py

@@ -67,6 +67,10 @@ def test_auto_create_destination_attributes(mocker, db):
             'fyle_integrations_platform_connector.apis.ExpenseCustomFields.post',
             return_value=[]
         )
+        mocker.patch(
+            'qbosdk.apis.Classes.count',
+            return_value=10
+        )
         mocker.patch(
             'qbosdk.apis.Classes.get_all_generator',
             return_value=[expense_custom_field_data['create_new_auto_create_expense_custom_fields_destination_attributes']]
@@ -103,6 +107,10 @@ def test_auto_create_destination_attributes(mocker, db):
             'fyle_integrations_platform_connector.apis.ExpenseCustomFields.get_by_id',
             return_value=expense_custom_field_data['create_new_auto_create_expense_custom_fields_get_by_id']
         )
+        mocker.patch(
+            'qbosdk.apis.Classes.count',
+            return_value=10
+        )
         mocker.patch(
             'qbosdk.apis.Classes.get_all_generator',
             return_value=[expense_custom_field_data['create_new_auto_create_expense_custom_fields_destination_attributes_subsequent_run']]