Import Logs for Deps Import

apps/fyle/serializers.py

@@ -15,7 +15,7 @@
 from apps.workspaces.models import Workspace, FyleCredential
 from apps.fyle.models import ExpenseFilter, DependentFieldSetting
 from apps.fyle.helpers import get_expense_fields
-
+from apps.mappings.imports.queues import chain_import_fields_to_fyle
 
 logger = logging.getLogger(__name__)
 logger.level = logging.INFO
@@ -39,6 +39,7 @@ def create(self, validated_data):
             platform = PlatformConnector(fyle_credentials)
 
             if refresh:
+                chain_import_fields_to_fyle(workspace_id=workspace_id)
                 platform.import_fyle_dimensions()
                 workspace.source_synced_at = datetime.now()
                 workspace.save(update_fields=['source_synced_at'])

apps/mappings/exceptions.py

@@ -17,8 +17,12 @@
 
 
 def handle_import_exceptions(func):
-    def new_fn(expense_attribute_instance, *args):
-        import_log: ImportLog = args[0]
+    def new_fn(expense_attribute_instance, *args, **kwargs):
+        import_log = None
+        if isinstance(expense_attribute_instance, ImportLog):
+            import_log: ImportLog = expense_attribute_instance
+        else:
+            import_log: ImportLog = args[0]
         workspace_id = import_log.workspace_id
         attribute_type = import_log.attribute_type
         error = {
@@ -28,7 +32,7 @@ def new_fn(expense_attribute_instance, *args):
             'response': None
         }
         try:
-            return func(expense_attribute_instance, *args)
+            return func(expense_attribute_instance, *args, **kwargs)
         except WrongParamsError as exception:
             error['message'] = exception.message
             error['response'] = exception.response

apps/mappings/imports/modules/base.py

@@ -1,4 +1,5 @@
 import math
+import logging
 from typing import List
 from datetime import (
     datetime,
@@ -20,6 +21,10 @@
 from apps.accounting_exports.models import Error
 
 
+logger = logging.getLogger(__name__)
+logger.level = logging.INFO
+
+
 class Base:
     """
     The Base class for all the modules
@@ -299,6 +304,8 @@ def post_to_fyle_and_sync(self, fyle_payload: List[object], resource_class, is_l
         :param is_last_batch: bool
         :param import_log: ImportLog object
         """
+        logger.info("| Importing {} to Fyle | Content: {{WORKSPACE_ID: {} Fyle Payload count: {} is_last_batch: {}}}".format(self.destination_field, self.workspace_id, len(fyle_payload), is_last_batch))
+
         if fyle_payload and self.platform_class_name in ['expense_custom_fields', 'merchants']:
             resource_class.post(fyle_payload)
         elif fyle_payload:

apps/mappings/imports/tasks.py

@@ -49,9 +49,12 @@ def auto_import_and_map_fyle_fields(workspace_id):
 
     chain = Chain()
 
+    cost_code_import_log = ImportLog.create_import_log('COST_CODE', workspace_id)
+    cost_category_import_log = ImportLog.create_import_log('COST_CATEGORY', workspace_id)
+
     chain.append('apps.mappings.tasks.sync_sage300_attributes', 'JOB', workspace_id)
-    chain.append('apps.mappings.tasks.sync_sage300_attributes', 'COST_CODE', workspace_id)
-    chain.append('apps.mappings.tasks.sync_sage300_attributes', 'COST_CATEGORY', workspace_id)
+    chain.append('apps.mappings.tasks.sync_sage300_attributes', 'COST_CODE', workspace_id, cost_code_import_log)
+    chain.append('apps.mappings.tasks.sync_sage300_attributes', 'COST_CATEGORY', workspace_id, cost_category_import_log)
     chain.append('apps.sage300.dependent_fields.import_dependent_fields_to_fyle', workspace_id)
 
     if import_log and import_log.status != 'COMPLETE':

apps/mappings/models.py

@@ -35,6 +35,20 @@ class Meta:
         db_table = 'import_logs'
         unique_together = ('workspace', 'attribute_type')
 
+    @classmethod
+    def create_import_log(self, attribute_type, workspace_id):
+        """
+        Create import logs set to IN_PROGRESS
+        """
+        import_log, _ = self.objects.update_or_create(
+            workspace_id=workspace_id,
+            attribute_type=attribute_type,
+            defaults={
+                'status': 'IN_PROGRESS'
+            }
+        )
+        return import_log
+
 
 class Version(BaseModel):
     """

apps/mappings/tasks.py

@@ -1,8 +1,9 @@
 from apps.workspaces.models import Sage300Credential
 from apps.sage300.utils import SageDesktopConnector
+from apps.mappings.models import ImportLog
 
 
-def sync_sage300_attributes(sage300_attribute_type: str, workspace_id: int):
+def sync_sage300_attributes(sage300_attribute_type: str, workspace_id: int, import_log: ImportLog = None):
     sage300_credentials: Sage300Credential = Sage300Credential.objects.get(workspace_id=workspace_id)
 
     sage300_connection = SageDesktopConnector(
@@ -12,8 +13,8 @@ def sync_sage300_attributes(sage300_attribute_type: str, workspace_id: int):
 
     sync_functions = {
         'JOB': sage300_connection.sync_jobs,
-        'COST_CODE': sage300_connection.sync_cost_codes,
-        'COST_CATEGORY': sage300_connection.sync_cost_categories,
+        'COST_CODE': lambda:sage300_connection.sync_cost_codes(import_log),
+        'COST_CATEGORY': lambda:sage300_connection.sync_cost_categories(import_log),
         'ACCOUNT': sage300_connection.sync_accounts,
         'VENDOR': sage300_connection.sync_vendors,
         'COMMITMENT': sage300_connection.sync_commitments,

apps/sage300/dependent_fields.py

@@ -11,6 +11,8 @@
 from apps.fyle.models import DependentFieldSetting
 from apps.sage300.models import CostCategory
 from apps.fyle.helpers import connect_to_platform
+from apps.mappings.models import ImportLog
+from apps.mappings.exceptions import handle_import_exceptions
 
 logger = logging.getLogger(__name__)
 logger.level = logging.INFO
@@ -68,7 +70,8 @@ def create_dependent_custom_field_in_fyle(workspace_id: int, fyle_attribute_type
     return platform.expense_custom_fields.post(expense_custom_field_payload)
 
 
-def post_dependent_cost_code(dependent_field_setting: DependentFieldSetting, platform: PlatformConnector, filters: Dict, is_enabled: bool = True) -> List[str]:
+@handle_import_exceptions
+def post_dependent_cost_code(import_log: ImportLog, dependent_field_setting: DependentFieldSetting, platform: PlatformConnector, filters: Dict, is_enabled: bool = True) -> List[str]:
     projects = CostCategory.objects.filter(**filters).values('job_name').annotate(cost_codes=ArrayAgg('cost_code_name', distinct=True))
     projects_from_categories = [project['job_name'] for project in projects]
     posted_cost_codes = []
@@ -103,10 +106,14 @@ def post_dependent_cost_code(dependent_field_setting: DependentFieldSetting, pla
                 logger.error(f'Exception while posting dependent cost code | Error: {exception} | Payload: {payload}')
                 raise
 
+    import_log.status = 'COMPLETE'
+    import_log.error_log = []
+    import_log.save()
     return posted_cost_codes
 
 
-def post_dependent_cost_type(dependent_field_setting: DependentFieldSetting, platform: PlatformConnector, filters: Dict):
+@handle_import_exceptions
+def post_dependent_cost_type(import_log: ImportLog, dependent_field_setting: DependentFieldSetting, platform: PlatformConnector, filters: Dict):
     cost_categories = CostCategory.objects.filter(is_imported=False, **filters).values('cost_code_name').annotate(cost_categories=ArrayAgg('name', distinct=True))
 
     for category in cost_categories:
@@ -129,6 +136,10 @@ def post_dependent_cost_type(dependent_field_setting: DependentFieldSetting, pla
                 logger.error(f'Exception while posting dependent cost type | Error: {exception} | Payload: {payload}')
                 raise
 
+    import_log.status = 'COMPLETE'
+    import_log.error_log = []
+    import_log.save()
+
 
 def post_dependent_expense_field_values(workspace_id: int, dependent_field_setting: DependentFieldSetting, platform: PlatformConnector = None):
     if not platform:
@@ -141,10 +152,20 @@ def post_dependent_expense_field_values(workspace_id: int, dependent_field_setti
     if dependent_field_setting.last_successful_import_at:
         filters['updated_at__gte'] = dependent_field_setting.last_successful_import_at
 
-    posted_cost_types = post_dependent_cost_code(dependent_field_setting, platform, filters)
+    cost_code_import_log = ImportLog.objects.filter(workspace_id=workspace_id, attribute_type='COST_CODE').first()
+    cost_category_import_log = ImportLog.objects.filter(workspace_id=workspace_id, attribute_type='COST_CATEGORY').first()
+
+    posted_cost_types = post_dependent_cost_code(cost_code_import_log, dependent_field_setting, platform, filters)
     if posted_cost_types:
         filters['cost_code_name__in'] = posted_cost_types
-        post_dependent_cost_type(dependent_field_setting, platform, filters)
+
+    if cost_code_import_log.status in ['FAILED', 'FATAL']:
+        cost_category_import_log.status = 'FAILED'
+        cost_category_import_log.error_log = {'message': 'Importing COST_CODE failed'}
+        cost_category_import_log.save()
+        return
+    else:
+        post_dependent_cost_type(cost_category_import_log, dependent_field_setting, platform, filters)
 
     DependentFieldSetting.objects.filter(workspace_id=workspace_id).update(last_successful_import_at=datetime.now())
 

apps/sage300/helpers.py

@@ -11,6 +11,7 @@
 from apps.sage300.models import CostCategory
 from apps.fyle.models import DependentFieldSetting
 from apps.sage300.dependent_fields import post_dependent_cost_code
+from apps.mappings.models import ImportLog
 
 logger = logging.getLogger(__name__)
 logger.level = logging.INFO
@@ -134,9 +135,10 @@ def update_and_disable_cost_code(workspace_id: int, cost_codes_to_disable: Dict,
             'job_id__in':list(cost_codes_to_disable.keys()),
             'workspace_id': workspace_id
         }
+        cost_code_import_log = ImportLog.create_import_log('COST_CODE', workspace_id)
 
         # This call will disable the cost codes in Fyle that has old project name
-        posted_cost_codes = post_dependent_cost_code(dependent_field_setting, platform, filters, is_enabled=False)
+        posted_cost_codes = post_dependent_cost_code(cost_code_import_log, dependent_field_setting, platform, filters, is_enabled=False)
 
         logger.info(f"Disabled Cost Codes in Fyle | WORKSPACE_ID: {workspace_id} | COUNT: {len(posted_cost_codes)}")
 

apps/sage300/utils.py

@@ -5,6 +5,7 @@
 from sage_desktop_sdk.sage_desktop_sdk import SageDesktopSDK
 from apps.sage300.models import CostCategory
 from apps.mappings.models import Version
+from apps.mappings.exceptions import handle_import_exceptions
 
 logger = logging.getLogger(__name__)
 logger.level = logging.INFO
@@ -262,7 +263,8 @@ def sync_commitment_items(self):
             ]
             self._sync_data(commitment_items, 'COMMITMENT_ITEM', 'commitment_item', self.workspace_id, field_names, False)
 
-    def sync_cost_codes(self):
+    @handle_import_exceptions
+    def sync_cost_codes(self, _import_log = None):
         """
         Synchronize cost codes from Sage Desktop SDK to your application
         """
@@ -272,7 +274,8 @@ def sync_cost_codes(self):
         self._sync_data(cost_codes, 'COST_CODE', 'cost_code', self.workspace_id, field_names)
         return []
 
-    def sync_cost_categories(self):
+    @handle_import_exceptions
+    def sync_cost_categories(self, import_log = None):
         """
          Synchronize categories from Sage Desktop SDK to your application
         """

requirements.txt

@@ -28,7 +28,7 @@ fyle==0.36.1
 
 # Reusable Fyle Packages
 fyle-rest-auth==1.7.2
-fyle-accounting-mappings==1.33.0
+fyle-accounting-mappings==1.33.1
 fyle-integrations-platform-connector==1.36.3
 
 

tests/test_fyle/test_views.py

@@ -10,6 +10,7 @@
 
 def test_import_fyle_attributes(mocker, api_client, test_connection, create_temp_workspace, add_fyle_credentials):
     mocker.patch('fyle_integrations_platform_connector.fyle_integrations_platform_connector.PlatformConnector.import_fyle_dimensions', return_value=[])
+    mocker.patch('apps.fyle.serializers.chain_import_fields_to_fyle', return_value=None)
 
     url = reverse('import-fyle-attributes', kwargs={'workspace_id': 1})
 

tests/test_sage300/test_dependent_fields.py

@@ -6,6 +6,7 @@
     import_dependent_fields_to_fyle
 )
 from apps.fyle.models import DependentFieldSetting
+from apps.mappings.models import ImportLog
 
 
 def test_construct_custom_field_placeholder():
@@ -64,14 +65,24 @@ def test_post_dependent_cost_code(
     }
 
     dependent_field_settings = DependentFieldSetting.objects.get(workspace_id=workspace_id)
+    cost_code_import_log = ImportLog.create_import_log('COST_CODE', workspace_id)
 
     result = post_dependent_cost_code(
+        cost_code_import_log,
         dependent_field_setting=dependent_field_settings,
         platform=platform.return_value,
         filters=filters
     )
 
     assert result == ['Direct Mail Campaign', 'Platform APIs']
+    assert cost_code_import_log.status == 'COMPLETE'
+
+    post_dependent_cost_code(
+        cost_code_import_log,
+        dependent_field_setting=dependent_field_settings,
+        platform=platform.return_value
+    )
+    assert cost_code_import_log.status == 'FATAL'
 
 
 def test_post_dependent_cost_type(
@@ -96,13 +107,24 @@ def test_post_dependent_cost_type(
 
     dependent_field_settings = DependentFieldSetting.objects.get(workspace_id=workspace_id)
 
+    cost_category_import_log = ImportLog.create_import_log('COST_CATEGORY', workspace_id)
+
     post_dependent_cost_type(
+        cost_category_import_log,
         dependent_field_setting=dependent_field_settings,
         platform=platform.return_value,
         filters=filters
     )
 
     assert platform.return_value.dependent_fields.bulk_post_dependent_expense_field_values.call_count == 2
+    assert cost_category_import_log.status == 'COMPLETE'
+
+    post_dependent_cost_type(
+        cost_category_import_log,
+        dependent_field_setting=dependent_field_settings,
+        platform=platform.return_value
+    )
+    assert cost_category_import_log.status == 'FATAL'
 
 
 def test_post_dependent_expense_field_values(
@@ -124,6 +146,9 @@ def test_post_dependent_expense_field_values(
 
     dependent_field_settings = DependentFieldSetting.objects.get(workspace_id=workspace_id)
 
+    ImportLog.create_import_log('COST_CODE', workspace_id)
+    ImportLog.create_import_log('COST_CATEGORY', workspace_id)
+
     post_dependent_expense_field_values(
         workspace_id=workspace_id,
         dependent_field_setting=dependent_field_settings
@@ -150,6 +175,9 @@ def test_import_dependent_fields_to_fyle(
         'dependent_fields.bulk_post_dependent_expense_field_values'
     )
 
+    ImportLog.create_import_log('COST_CODE', workspace_id)
+    ImportLog.create_import_log('COST_CATEGORY', workspace_id)
+
     import_dependent_fields_to_fyle(workspace_id)
 
     assert platform.return_value.dependent_fields.bulk_post_dependent_expense_field_values.call_count == 4

tests/test_sage300/test_utils.py

@@ -2,6 +2,7 @@
 from fyle_accounting_mappings.models import DestinationAttribute
 from apps.mappings.models import Version
 from apps.workspaces.models import Workspace
+from apps.mappings.models import ImportLog
 
 
 def test_sage_desktop_connector(
@@ -453,6 +454,8 @@ def test_sync_cost_categories(
         workspace_id=workspace_id
     )
 
+    cost_category_import_log = ImportLog.create_import_log('COST_CATEGORY', workspace_id)
+
     mock_category = [{
         "Id": 1,
         "JobId": "10064",
@@ -480,7 +483,9 @@ def test_sync_cost_categories(
 
     sage_connector.connection.categories.get_all_categories.return_value = categories_generator
 
-    sage_connector.sync_cost_categories()
+    sage_connector.sync_cost_categories(cost_category_import_log)
+
+    assert Version.objects.get(workspace_id=workspace_id).cost_category == 2
 
     assert Version.objects.get(workspace_id=workspace_id).cost_category == 2
 
@@ -501,6 +506,8 @@ def test_sync_cost_codes(
         workspace_id=workspace_id
     )
 
+    cost_code_import_log = ImportLog.create_import_log('COST_CODE', workspace_id)
+
     Version.objects.update_or_create(
         workspace_id=workspace_id,
         cost_code=1,
@@ -520,5 +527,5 @@ def test_sync_cost_codes(
 
     sage_connector.connection.cost_codes.get_all_costcodes.return_value = [[[mock_data]]]
 
-    result = sage_connector.sync_cost_codes()
+    result = sage_connector.sync_cost_codes(cost_code_import_log)
     assert result == []