From bffcf72335806f57de80d9ea253fdf611d9822de Mon Sep 17 00:00:00 2001 From: Ashutosh singh <55102089+Ashutosh619-sudo@users.noreply.github.com> Date: Tue, 8 Oct 2024 15:30:33 +0530 Subject: [PATCH] feat: Added limits to attributes (#649) * feat: Added limits to attributes * resolved comments and added test * added test * renamed function * fixed test --- apps/netsuite/connector.py | 172 ++++++++++++++++---------- requirements.txt | 2 +- tests/test_mappings/test_tasks.py | 5 + tests/test_netsuite/test_connector.py | 108 +++++++++++++++- 4 files changed, 222 insertions(+), 65 deletions(-) diff --git a/apps/netsuite/connector.py b/apps/netsuite/connector.py index 4ca7025a..65f6cad2 100644 --- a/apps/netsuite/connector.py +++ b/apps/netsuite/connector.py @@ -1,6 +1,7 @@ import re import json from datetime import datetime, timedelta +from django.utils import timezone from typing import List, Dict import logging @@ -29,8 +30,13 @@ logger.level = logging.INFO SYNC_UPPER_LIMIT = { - 'projects': 25000, - 'customers': 25000 + 'projects': 10000, + 'customers': 25000, + 'classes': 2000, + 'accounts': 2000, + 'locations': 2000, + 'departments': 2000, + 'vendors': 20000, } @@ -76,11 +82,31 @@ def get_tax_code_name(item_id, tax_type, rate): return '{0}: {1} @{2}%'.format(tax_type, item_id, rate) else: return '{0} @{1}%'.format(item_id, rate) + + def is_sync_allowed(self, attribute_type: str, attribute_count: int): + """ + Checks if the sync is allowed + + Returns: + bool: True + """ + if attribute_count > SYNC_UPPER_LIMIT[attribute_type]: + workspace_created_at = Workspace.objects.get(id=self.workspace_id).created_at + if workspace_created_at > timezone.make_aware(datetime(2024, 10, 1), timezone.get_current_timezone()): + return False + else: + return True + + return True def sync_accounts(self): """ Sync accounts """ + attribute_count = self.connection.accounts.count() + if not self.is_sync_allowed(attribute_type = 'accounts', attribute_count=attribute_count): + logger.info('Skipping sync of accounts for workspace %s as it has %s counts which is over the limit', self.workspace_id, attribute_count) + return accounts_generator = self.connection.accounts.get_all_generator() for accounts in accounts_generator: attributes = { @@ -517,6 +543,11 @@ def sync_locations(self): """ Sync locations """ + attribute_count = self.connection.locations.count() + if not self.is_sync_allowed(attribute_type = 'locations', attribute_count = attribute_count): + logger.info('Skipping sync of locations for workspace %s as it has %s counts which is over the limit', self.workspace_id, attribute_count) + return + subsidiary_mapping = SubsidiaryMapping.objects.get(workspace_id=self.workspace_id) location_generator = self.connection.locations.get_all_generator() @@ -556,6 +587,11 @@ def sync_classifications(self): """ Sync classification """ + attribute_count = self.connection.classifications.count() + if not self.is_sync_allowed(attribute_type = 'classes', attribute_count = attribute_count): + logger.info('Skipping sync of classes for workspace %s as it has %s counts which is over the limit', self.workspace_id, attribute_count) + return + classification_generator = self.connection.classifications.get_all_generator() classification_attributes = [] @@ -580,6 +616,10 @@ def sync_departments(self): """ Sync departments """ + attribute_count = self.connection.departments.count() + if not self.is_sync_allowed(attribute_type = 'departments', attribute_count = attribute_count): + logger.info('Skipping sync of department for workspace %s as it has %s counts which is over the limit', self.workspace_id, attribute_count) + return department_generator = self.connection.departments.get_all_generator() department_attributes = [] @@ -604,6 +644,11 @@ def sync_vendors(self): """ Sync vendors """ + attribute_count = self.connection.vendors.count() + if not self.is_sync_allowed(attribute_type = 'vendors', attribute_count=attribute_count): + logger.info('Skipping sync of vendors for workspace %s as it has %s counts which is over the limit', self.workspace_id, attribute_count) + return + subsidiary_mapping = SubsidiaryMapping.objects.get(workspace_id=self.workspace_id) configuration = Configuration.objects.filter(workspace_id=self.workspace_id).first() if not configuration: @@ -1036,40 +1081,41 @@ def sync_projects(self): """ Sync projects """ - projects_count = self.connection.projects.count() - - if projects_count <= SYNC_UPPER_LIMIT['projects']: - projects_generator = self.connection.projects.get_all_generator() - - for projects in projects_generator: - attributes = [] - destination_ids = DestinationAttribute.objects.filter( - workspace_id=self.workspace_id, - attribute_type= 'PROJECT', - display_name='Project' - ).values_list('destination_id', flat=True) + attribute_count = self.connection.projects.count() + if not self.is_sync_allowed(attribute_type = 'projects', attribute_count = attribute_count): + logger.info('Skipping sync of projects for workspace %s as it has %s counts which is over the limit', self.workspace_id, attribute_count) + return + + projects_generator = self.connection.projects.get_all_generator() + for projects in projects_generator: + attributes = [] + destination_ids = DestinationAttribute.objects.filter( + workspace_id=self.workspace_id, + attribute_type= 'PROJECT', + display_name='Project' + ).values_list('destination_id', flat=True) - for project in projects: - value = self.__decode_project_or_customer_name(project['entityId']) + for project in projects: + value = self.__decode_project_or_customer_name(project['entityId']) - if project['internalId'] in destination_ids : - attributes.append({ - 'attribute_type': 'PROJECT', - 'display_name': 'Project', - 'value': value, - 'destination_id': project['internalId'], - 'active': not project['isInactive'] - }) - elif not project['isInactive']: - attributes.append({ - 'attribute_type': 'PROJECT', - 'display_name': 'Project', - 'value': value, - 'destination_id': project['internalId'], - 'active': True - }) - DestinationAttribute.bulk_create_or_update_destination_attributes( - attributes, 'PROJECT', self.workspace_id, True) + if project['internalId'] in destination_ids : + attributes.append({ + 'attribute_type': 'PROJECT', + 'display_name': 'Project', + 'value': value, + 'destination_id': project['internalId'], + 'active': not project['isInactive'] + }) + elif not project['isInactive']: + attributes.append({ + 'attribute_type': 'PROJECT', + 'display_name': 'Project', + 'value': value, + 'destination_id': project['internalId'], + 'active': True + }) + DestinationAttribute.bulk_create_or_update_destination_attributes( + attributes, 'PROJECT', self.workspace_id, True) return [] @@ -1077,36 +1123,38 @@ def sync_customers(self): """ Sync customers """ - customers_count = self.connection.customers.count() - - if customers_count <= SYNC_UPPER_LIMIT['customers']: - customers_generator = self.connection.customers.get_all_generator() + attribute_count = self.connection.customers.count() + if not self.is_sync_allowed(attribute_type = 'customers', attribute_count = attribute_count): + logger.info('Skipping sync of customers for workspace %s as it has %s counts which is over the limit', self.workspace_id, attribute_count) + return + + customers_generator = self.connection.customers.get_all_generator() - for customers in customers_generator: - attributes = [] - destination_ids = DestinationAttribute.objects.filter(workspace_id=self.workspace_id,\ - attribute_type= 'PROJECT', display_name='Customer').values_list('destination_id', flat=True) - for customer in customers: - value = self.__decode_project_or_customer_name(customer['entityId']) - if customer['internalId'] in destination_ids : - attributes.append({ - 'attribute_type': 'PROJECT', - 'display_name': 'Customer', - 'value': value, - 'destination_id': customer['internalId'], - 'active': not customer['isInactive'] - }) - elif not customer['isInactive']: - attributes.append({ - 'attribute_type': 'PROJECT', - 'display_name': 'Customer', - 'value': value, - 'destination_id': customer['internalId'], - 'active': True - }) + for customers in customers_generator: + attributes = [] + destination_ids = DestinationAttribute.objects.filter(workspace_id=self.workspace_id,\ + attribute_type= 'PROJECT', display_name='Customer').values_list('destination_id', flat=True) + for customer in customers: + value = self.__decode_project_or_customer_name(customer['entityId']) + if customer['internalId'] in destination_ids : + attributes.append({ + 'attribute_type': 'PROJECT', + 'display_name': 'Customer', + 'value': value, + 'destination_id': customer['internalId'], + 'active': not customer['isInactive'] + }) + elif not customer['isInactive']: + attributes.append({ + 'attribute_type': 'PROJECT', + 'display_name': 'Customer', + 'value': value, + 'destination_id': customer['internalId'], + 'active': True + }) - DestinationAttribute.bulk_create_or_update_destination_attributes( - attributes, 'PROJECT', self.workspace_id, True) + DestinationAttribute.bulk_create_or_update_destination_attributes( + attributes, 'PROJECT', self.workspace_id, True) return [] diff --git a/requirements.txt b/requirements.txt index 7df1c37b..0998893e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,7 +33,7 @@ isort==5.10.1 lazy-object-proxy==1.6.0 lxml==4.6.5 mccabe==0.6.1 -netsuitesdk==2.21.3 +netsuitesdk==2.23.0 oauthlib==3.2.1 packaging==21.3 platformdirs==2.4.0 diff --git a/tests/test_mappings/test_tasks.py b/tests/test_mappings/test_tasks.py index fda2f96b..e023bb7d 100644 --- a/tests/test_mappings/test_tasks.py +++ b/tests/test_mappings/test_tasks.py @@ -93,6 +93,11 @@ def test_remove_duplicates(db): def test_async_auto_map_employees(mocker, db): + mocker.patch( + 'netsuitesdk.api.vendors.Vendors.count', + return_value=5 + ) + mocker.patch( 'netsuitesdk.api.vendors.Vendors.get_all_generator', return_value=netsuite_data['get_all_vendors'] diff --git a/tests/test_netsuite/test_connector.py b/tests/test_netsuite/test_connector.py index 9eb0ea7a..8531a7e0 100644 --- a/tests/test_netsuite/test_connector.py +++ b/tests/test_netsuite/test_connector.py @@ -1,7 +1,8 @@ import pytest +from datetime import datetime from unittest import mock from apps.fyle.models import ExpenseGroup -from fyle_accounting_mappings.models import DestinationAttribute, ExpenseAttribute +from fyle_accounting_mappings.models import DestinationAttribute, ExpenseAttribute, Mapping, CategoryMapping from apps.netsuite.connector import NetSuiteConnector, NetSuiteCredentials from apps.workspaces.models import Configuration, Workspace from netsuitesdk import NetSuiteRequestError @@ -132,9 +133,13 @@ def test_get_expense_report(mocker, db): assert dict_compare_keys(expense_report, data['get_expense_report_response'][0]) == [], 'get expense report returns diff in keys' def test_sync_vendors(mocker, db): + mocker.patch( + 'netsuitesdk.api.vendors.Vendors.count', + return_value=0 + ) mocker.patch( 'netsuitesdk.api.vendors.Vendors.get_all_generator', - return_value=data['get_all_vendors'] + return_value=data['get_all_vendors'] ) netsuite_credentials = NetSuiteCredentials.objects.get(workspace_id=1) netsuite_connection = NetSuiteConnector(netsuite_credentials=netsuite_credentials, workspace_id=1) @@ -193,6 +198,10 @@ def test_sync_employees(mocker, db): @pytest.mark.django_db() def test_sync_accounts(mocker, db): + mocker.patch( + 'netsuitesdk.api.accounts.Accounts.count', + return_value=5 + ) mocker.patch( 'netsuitesdk.api.accounts.Accounts.get_all_generator', return_value=data['get_all_accounts'] @@ -324,6 +333,10 @@ def test_sync_subsidiaries(mocker, db): assert subsidiaries == 8 def test_sync_locations(mocker, db): + mocker.patch( + 'netsuitesdk.api.locations.Locations.count', + return_value=5 + ) mocker.patch( 'netsuitesdk.api.locations.Locations.get_all_generator', return_value=data['get_all_locations'] @@ -341,6 +354,10 @@ def test_sync_locations(mocker, db): def test_sync_departments(mocker, db): + mocker.patch( + 'netsuitesdk.api.departments.Departments.count', + return_value=5 + ) mocker.patch( 'netsuitesdk.api.departments.Departments.get_all_generator', return_value=data['get_all_departments'] @@ -421,6 +438,10 @@ def test_sync_currencies(mocker, db): def test_sync_classifications(mocker, db): + mocker.patch( + 'netsuitesdk.api.classifications.Classifications.count', + return_value=5 + ) mocker.patch( 'netsuitesdk.api.classifications.Classifications.get_all_generator', return_value=data['get_all_classifications'] @@ -659,3 +680,86 @@ def test_update_destination_attributes(db, mocker): assert custom_type_destination_attribute.destination_id == '1' elif custom_type_destination_attribute.value == 'Type D': assert custom_type_destination_attribute.destination_id == '4' + + +def test_skip_sync_attributes(mocker, db): + mocker.patch( + 'netsuitesdk.api.projects.Projects.count', + return_value=10001 + ) + + mocker.patch( + 'netsuitesdk.api.classifications.Classifications.count', + return_value=2001 + ) + mocker.patch( + 'netsuitesdk.api.accounts.Accounts.count', + return_value=2001 + ) + mocker.patch( + 'netsuitesdk.api.locations.Locations.count', + return_value=2001 + ) + mocker.patch( + 'netsuitesdk.api.departments.Departments.count', + return_value=2001 + ) + mocker.patch( + 'netsuitesdk.api.customers.Customers.count', + return_value=25001 + ) + mocker.patch( + 'netsuitesdk.api.vendors.Vendors.count', + return_value=20001 + ) + + today = datetime.today() + Workspace.objects.filter(id=1).update(created_at=today) + netsuite_credentials = NetSuiteCredentials.objects.get(workspace_id=1) + netsuite_connection = NetSuiteConnector(netsuite_credentials=netsuite_credentials, workspace_id=1) + + Mapping.objects.filter(workspace_id=1).delete() + CategoryMapping.objects.filter(workspace_id=1).delete() + + DestinationAttribute.objects.filter(workspace_id=1, attribute_type='PROJECT').delete() + + netsuite_connection.sync_projects() + + new_project_count = DestinationAttribute.objects.filter(workspace_id=1, attribute_type='PROJECT').count() + assert new_project_count == 0 + + DestinationAttribute.objects.filter(workspace_id=1, attribute_type='CLASS').delete() + + netsuite_connection.sync_classifications() + + classifications = DestinationAttribute.objects.filter(attribute_type='CLASS', workspace_id=1).count() + assert classifications == 0 + + DestinationAttribute.objects.filter(workspace_id=1, attribute_type='ACCOUNT').delete() + + netsuite_connection.sync_accounts() + + new_project_count = DestinationAttribute.objects.filter(workspace_id=1, attribute_type='ACCOUNT').count() + assert new_project_count == 0 + + DestinationAttribute.objects.filter(workspace_id=1, attribute_type='LOCATION').delete() + + netsuite_connection.sync_locations() + + new_project_count = DestinationAttribute.objects.filter(workspace_id=1, attribute_type='LOCATION').count() + assert new_project_count == 0 + + DestinationAttribute.objects.filter(workspace_id=1, attribute_type='DEPARTMENT').delete() + + netsuite_connection.sync_departments() + + new_project_count = DestinationAttribute.objects.filter(workspace_id=1, attribute_type='DEPARTMENT').count() + assert new_project_count == 0 + + DestinationAttribute.objects.filter(workspace_id=1, attribute_type='CUSTOMER').delete() + + netsuite_connection.sync_customers() + + new_project_count = DestinationAttribute.objects.filter(workspace_id=1, attribute_type='CUSTOMER').count() + assert new_project_count == 0 + \ No newline at end of file