feat: Add extensive logger and limits (#167)

* feat: Add extensive logger and limits

* test fixed

* added settings for logger

* change to info

* bump up version

* added test

apps/business_central/utils.py

@@ -1,6 +1,8 @@
 import base64
 import logging
 from typing import Dict, List
+from datetime import datetime
+from django.utils import timezone
 
 from dynamics.core.client import Dynamics
 from fyle_accounting_mappings.models import DestinationAttribute
@@ -11,6 +13,12 @@
 logger = logging.getLogger(__name__)
 logger.level = logging.INFO
 
+SYNC_UPPER_LIMIT = {
+    'accounts': 2000,
+    'vendors': 10000,
+    'locations': 1000
+}
+
 
 class BusinessCentralConnector:
     """
@@ -58,6 +66,22 @@ def _create_destination_attribute(self, attribute_type, display_name, value, des
             'detail': detail
         }
 
+    def is_sync_allowed(self, attribute_type: str, attribute_count: int):
+        """
+        Checks if the sync is allowed
+
+        Returns:
+            bool: True
+        """
+        if attribute_count > SYNC_UPPER_LIMIT[attribute_type]:
+            workspace_created_at = Workspace.objects.get(id=self.workspace_id).created_at
+            if workspace_created_at > timezone.make_aware(datetime(2024, 10, 1), timezone.get_current_timezone()):
+                return False
+            else:
+                return True
+
+        return True
+
     def _sync_data(self, data, attribute_type, display_name, workspace_id, field_names):
         """
         Synchronize data from MS Dynamics SDK to your application
@@ -111,6 +135,10 @@ def sync_accounts(self):
         """
         Synchronize accounts from MS Dynamics SDK to your application
         """
+        attribute_count = self.connection.accounts.count()
+        if not self.is_sync_allowed(attribute_type = 'accounts', attribute_count=attribute_count):
+            logger.info('Skipping sync of accounts for workspace %s as it has %s counts which is over the limit', self.workspace_id, attribute_count)
+            return
         field_names = ['category', 'subCategory', 'accountType', 'directPosting', 'lastModifiedDateTime']
 
         accounts = self.connection.accounts.get_all()
@@ -121,6 +149,10 @@ def sync_vendors(self):
         """
         Synchronize vendors from MS Dynamics SDK to your application
         """
+        attribute_count = self.connection.vendors.count()
+        if not self.is_sync_allowed(attribute_type = 'vendors', attribute_count=attribute_count):
+            logger.info('Skipping sync of vendors for workspace %s as it has %s counts which is over the limit', self.workspace_id, attribute_count)
+            return
         field_names = ['email', 'currencyId', 'currencyCode', 'lastModifiedDateTime']
 
         vendors = self.connection.vendors.get_all()
@@ -141,6 +173,10 @@ def sync_locations(self):
         """
         Synchronize locations from MS Dynamics SDK to your application
         """
+        attribute_count = self.connection.locations.count()
+        if not self.is_sync_allowed(attribute_type = 'locations', attribute_count = attribute_count):
+            logger.info('Skipping sync of locations for workspace %s as it has %s counts which is over the limit', self.workspace_id, attribute_count)
+            return
         field_names = ['code', 'city', 'country']
 
         locations = self.connection.locations.get_all()

ms_business_central_api/settings.py

@@ -128,7 +128,11 @@
         'requests': {
             'format': 'request {levelname} %s {asctime} {message}' % SERVICE_NAME,
             'style': '{'
-        }
+        },
+        "standard": {
+            "format": "{levelname} %s {asctime} {name} {message}" % SERVICE_NAME,
+            "style": "{",
+        },
     },
     'handlers': {
         'debug_logs': {
@@ -141,6 +145,10 @@
             'stream': sys.stdout,
             'formatter': 'requests'
         },
+        "console": {
+            "class": "logging.StreamHandler",
+            "formatter": "standard",
+        },
     },
     'loggers': {
         'django': {
@@ -173,7 +181,12 @@
             'handlers': ['request_logs'],
             'level': 'INFO',
             'propagate': False
-        }
+        },
+        "dynamics.apis.api_base": {
+            "handlers": ["console"],
+            "level": "INFO",
+            "propagate": True
+        },
     }
 }
 

requirements.txt

@@ -25,10 +25,10 @@ gevent==23.9.1
 gunicorn==20.1.0
 
 # Platform SDK
-fyle==0.37.0
+fyle==0.37.2
 
 # Business central sdk
-ms-dynamics-business-central-sdk==1.4.2
+ms-dynamics-business-central-sdk==1.5.2
 
 # Reusable Fyle Packages
 fyle-rest-auth==1.7.2

tests/test_business_central/test_utils.py

@@ -1,3 +1,8 @@
+from apps.workspaces.models import Workspace
+from datetime import datetime
+from fyle_accounting_mappings.models import DestinationAttribute, Mapping, CategoryMapping
+
+
 def test_post_attachments(
     db,
     mocker,
@@ -143,3 +148,50 @@ def test_bulk_post_journal_lineitems(
     assert len(response) == 2
     assert response[0]['id'] == 'Journal_Line_Item_Id'
     assert response[1]['id'] == 'Journal_Line_Item_Id_2'
+
+
+def test_skip_sync_attributes(mocker, db, create_business_central_connection):
+
+    today = datetime.today()
+    Workspace.objects.filter(id=1).update(created_at=today)
+    business_central_connection = create_business_central_connection
+
+    Mapping.objects.filter(workspace_id=1).delete()
+    CategoryMapping.objects.filter(workspace_id=1).delete()
+
+    DestinationAttribute.objects.filter(workspace_id=1, attribute_type='VENDOR').delete()
+
+    mocker.patch.object(
+        business_central_connection.connection.vendors,
+        'count',
+        return_value=10001
+    )
+
+    business_central_connection.sync_vendors()
+
+    vendors = DestinationAttribute.objects.filter(attribute_type='VENDOR', workspace_id=1).count()
+    assert vendors == 0
+
+    DestinationAttribute.objects.filter(workspace_id=1, attribute_type='LOCATION').delete()
+    mocker.patch.object(
+        business_central_connection.connection.locations,
+        'count',
+        return_value=1001
+    )
+
+    business_central_connection.sync_locations()
+
+    locations = DestinationAttribute.objects.filter(attribute_type='LOCATION', workspace_id=1).count()
+    assert locations == 0
+
+    DestinationAttribute.objects.filter(workspace_id=1, attribute_type='ACCOUNT').delete()
+    mocker.patch.object(
+        business_central_connection.connection.accounts,
+        'count',
+        return_value=2001
+    )
+
+    business_central_connection.sync_accounts()
+
+    accounts = DestinationAttribute.objects.filter(attribute_type='ACCOUNT', workspace_id=1).count()
+    assert accounts == 0

tests/test_mappings/test_imports/test_modules/test_base.py

@@ -27,6 +27,10 @@ def test_sync_destination_attributes(
         "dynamics.apis.Accounts.get_all",
         return_value=destination_attributes_data["get_account_destination_attributes"],
     )
+    mocker.patch(
+        "dynamics.apis.Accounts.count",
+        return_value=5,
+    )
 
     account_count = DestinationAttribute.objects.filter(
         workspace_id=workspace_id, attribute_type="ACCOUNT"

tests/test_mappings/test_imports/test_modules/test_categories.py

@@ -22,6 +22,10 @@ def test_sync_destination_attributes_categories(
         "dynamics.apis.Accounts.get_all",
         return_value=destination_attributes_data["get_account_destination_attributes"],
     )
+    mocker.patch(
+        "dynamics.apis.Accounts.count",
+        return_value=5,
+    )
 
     account_count = DestinationAttribute.objects.filter(
         workspace_id=workspace_id, attribute_type="ACCOUNT"
@@ -36,6 +40,10 @@ def test_sync_destination_attributes_categories(
     ).count()
     assert new_account_count == 2
 
+    mocker.patch(
+        "dynamics.apis.Locations.count",
+        return_value=5
+    )
     mocker.patch(
         "dynamics.apis.Locations.get_all",
         return_value=destination_attributes_data[
@@ -132,6 +140,10 @@ def test_auto_create_destination_attributes(
             "fyle_integrations_platform_connector.apis.Categories.post_bulk",
             return_value=[],
         )
+        mocker.patch(
+            "dynamics.apis.Accounts.count",
+            return_value=5
+        )
         mocker.patch(
             "dynamics.apis.Accounts.get_all",
             return_value=destination_attributes_data[

tests/test_mappings/test_imports/test_modules/test_cost_centers.py

@@ -84,6 +84,10 @@ def test_auto_create_destination_attributes(
             "fyle_integrations_platform_connector.apis.CostCenters.post_bulk",
             return_value=[],
         )
+        mocker.patch(
+            "dynamics.apis.Locations.count",
+            return_value=5,
+        )
         mocker.patch(
             "dynamics.apis.Locations.get_all",
             return_value=data["get_location_destination_attributes_0"],

tests/test_mappings/test_imports/test_modules/test_expense_custom_fields.py

@@ -87,6 +87,10 @@ def test_auto_create_destination_attributes(
             "fyle_integrations_platform_connector.apis.ExpenseCustomFields.post",
             return_value=[],
         )
+        mocker.patch(
+            "dynamics.apis.Locations.count",
+            return_value=5,
+        )
         mocker.patch(
             "dynamics.apis.Locations.get_all",
             return_value=data["get_location_destination_attributes_2"],
@@ -139,6 +143,10 @@ def test_auto_create_destination_attributes(
             "fyle_integrations_platform_connector.apis.ExpenseCustomFields.get_by_id",
             return_value=data["create_new_auto_create_expense_custom_fields_get_by_id"],
         )
+        mocker.patch(
+            "dynamics.apis.Locations.count",
+            return_value=5,
+        )
         mocker.patch(
             "dynamics.apis.Locations.get_all",
             return_value=data["get_location_destination_attributes_3"],

tests/test_mappings/test_imports/test_modules/test_merchants.py

@@ -61,6 +61,10 @@ def test_sync_destination_atrributes(
     add_business_central_creds,
     mocker,
 ):
+    mocker.patch(
+        "dynamics.apis.Vendors.count",
+        return_value=5,
+    )
     mocker.patch(
         "dynamics.apis.Vendors.get_all",
         return_value=data["get_vendors_destination_attributes"],
@@ -107,6 +111,10 @@ def test_auto_create_destination_attributes(
         mocker.patch(
             "fyle_integrations_platform_connector.apis.Merchants.post", return_value=[]
         )
+        mocker.patch(
+            "dynamics.apis.Vendors.count",
+            return_value=5,
+        )
         mocker.patch(
             "dynamics.apis.Vendors.get_all",
             return_value=data["get_vendors_destination_attributes"],
@@ -151,6 +159,10 @@ def test_auto_create_destination_attributes(
         mocker.patch(
             "fyle_integrations_platform_connector.apis.Merchants.post", return_value=[]
         )
+        mocker.patch(
+            "dynamics.apis.Vendors.count",
+            return_value=5,
+        )
         mocker.patch(
             "dynamics.apis.Vendors.get_all",
             return_value=data["get_vendors_destination_attributes_subsequent_run"],