diff --git a/apps/fyle/serializers.py b/apps/fyle/serializers.py index 65d54c9e..a95571dc 100644 --- a/apps/fyle/serializers.py +++ b/apps/fyle/serializers.py @@ -15,7 +15,7 @@ from apps.workspaces.models import Workspace, FyleCredential from apps.fyle.models import ExpenseFilter, DependentFieldSetting from apps.fyle.helpers import get_expense_fields - +from apps.mappings.imports.queues import chain_import_fields_to_fyle logger = logging.getLogger(__name__) logger.level = logging.INFO @@ -39,6 +39,7 @@ def create(self, validated_data): platform = PlatformConnector(fyle_credentials) if refresh: + chain_import_fields_to_fyle(workspace_id=workspace_id) platform.import_fyle_dimensions() workspace.source_synced_at = datetime.now() workspace.save(update_fields=['source_synced_at']) diff --git a/apps/mappings/exceptions.py b/apps/mappings/exceptions.py index 5c623297..560847ed 100644 --- a/apps/mappings/exceptions.py +++ b/apps/mappings/exceptions.py @@ -17,8 +17,12 @@ def handle_import_exceptions(func): - def new_fn(expense_attribute_instance, *args): - import_log: ImportLog = args[0] + def new_fn(expense_attribute_instance, *args, **kwargs): + import_log = None + if isinstance(expense_attribute_instance, ImportLog): + import_log: ImportLog = expense_attribute_instance + else: + import_log: ImportLog = args[0] workspace_id = import_log.workspace_id attribute_type = import_log.attribute_type error = { @@ -28,7 +32,7 @@ def new_fn(expense_attribute_instance, *args): 'response': None } try: - return func(expense_attribute_instance, *args) + return func(expense_attribute_instance, *args, **kwargs) except WrongParamsError as exception: error['message'] = exception.message error['response'] = exception.response diff --git a/apps/mappings/helpers.py b/apps/mappings/helpers.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/mappings/imports/modules/base.py b/apps/mappings/imports/modules/base.py index cf8d9a5e..7193d464 100644 --- a/apps/mappings/imports/modules/base.py +++ b/apps/mappings/imports/modules/base.py @@ -1,4 +1,5 @@ import math +import logging from typing import List from datetime import ( datetime, @@ -20,6 +21,10 @@ from apps.accounting_exports.models import Error +logger = logging.getLogger(__name__) +logger.level = logging.INFO + + class Base: """ The Base class for all the modules @@ -299,6 +304,8 @@ def post_to_fyle_and_sync(self, fyle_payload: List[object], resource_class, is_l :param is_last_batch: bool :param import_log: ImportLog object """ + logger.info("| Importing {} to Fyle | Content: {{WORKSPACE_ID: {} Fyle Payload count: {} is_last_batch: {}}}".format(self.destination_field, self.workspace_id, len(fyle_payload), is_last_batch)) + if fyle_payload and self.platform_class_name in ['expense_custom_fields', 'merchants']: resource_class.post(fyle_payload) elif fyle_payload: diff --git a/apps/mappings/imports/tasks.py b/apps/mappings/imports/tasks.py index c3e0a102..728054e4 100644 --- a/apps/mappings/imports/tasks.py +++ b/apps/mappings/imports/tasks.py @@ -49,9 +49,12 @@ def auto_import_and_map_fyle_fields(workspace_id): chain = Chain() + cost_code_import_log = ImportLog.create_import_log('COST_CODE', workspace_id) + cost_category_import_log = ImportLog.create_import_log('COST_CATEGORY', workspace_id) + chain.append('apps.mappings.tasks.sync_sage300_attributes', 'JOB', workspace_id) - chain.append('apps.mappings.tasks.sync_sage300_attributes', 'COST_CODE', workspace_id) - chain.append('apps.mappings.tasks.sync_sage300_attributes', 'COST_CATEGORY', workspace_id) + chain.append('apps.mappings.tasks.sync_sage300_attributes', 'COST_CODE', workspace_id, cost_code_import_log) + chain.append('apps.mappings.tasks.sync_sage300_attributes', 'COST_CATEGORY', workspace_id, cost_category_import_log) chain.append('apps.sage300.dependent_fields.import_dependent_fields_to_fyle', workspace_id) if import_log and import_log.status != 'COMPLETE': diff --git a/apps/mappings/models.py b/apps/mappings/models.py index accff5c0..95967734 100644 --- a/apps/mappings/models.py +++ b/apps/mappings/models.py @@ -35,6 +35,20 @@ class Meta: db_table = 'import_logs' unique_together = ('workspace', 'attribute_type') + @classmethod + def create_import_log(self, attribute_type, workspace_id): + """ + Create import logs set to IN_PROGRESS + """ + import_log, _ = self.objects.update_or_create( + workspace_id=workspace_id, + attribute_type=attribute_type, + defaults={ + 'status': 'IN_PROGRESS' + } + ) + return import_log + class Version(BaseModel): """ diff --git a/apps/mappings/tasks.py b/apps/mappings/tasks.py index 59551db5..87a80f76 100644 --- a/apps/mappings/tasks.py +++ b/apps/mappings/tasks.py @@ -1,8 +1,9 @@ from apps.workspaces.models import Sage300Credential from apps.sage300.utils import SageDesktopConnector +from apps.mappings.models import ImportLog -def sync_sage300_attributes(sage300_attribute_type: str, workspace_id: int): +def sync_sage300_attributes(sage300_attribute_type: str, workspace_id: int, import_log: ImportLog = None): sage300_credentials: Sage300Credential = Sage300Credential.objects.get(workspace_id=workspace_id) sage300_connection = SageDesktopConnector( @@ -12,8 +13,8 @@ def sync_sage300_attributes(sage300_attribute_type: str, workspace_id: int): sync_functions = { 'JOB': sage300_connection.sync_jobs, - 'COST_CODE': sage300_connection.sync_cost_codes, - 'COST_CATEGORY': sage300_connection.sync_cost_categories, + 'COST_CODE': lambda:sage300_connection.sync_cost_codes(import_log), + 'COST_CATEGORY': lambda:sage300_connection.sync_cost_categories(import_log), 'ACCOUNT': sage300_connection.sync_accounts, 'VENDOR': sage300_connection.sync_vendors, 'COMMITMENT': sage300_connection.sync_commitments, diff --git a/apps/sage300/dependent_fields.py b/apps/sage300/dependent_fields.py index 0cdc0c00..0dacb17b 100644 --- a/apps/sage300/dependent_fields.py +++ b/apps/sage300/dependent_fields.py @@ -11,6 +11,8 @@ from apps.fyle.models import DependentFieldSetting from apps.sage300.models import CostCategory from apps.fyle.helpers import connect_to_platform +from apps.mappings.models import ImportLog +from apps.mappings.exceptions import handle_import_exceptions logger = logging.getLogger(__name__) logger.level = logging.INFO @@ -68,7 +70,8 @@ def create_dependent_custom_field_in_fyle(workspace_id: int, fyle_attribute_type return platform.expense_custom_fields.post(expense_custom_field_payload) -def post_dependent_cost_code(dependent_field_setting: DependentFieldSetting, platform: PlatformConnector, filters: Dict, is_enabled: bool = True) -> List[str]: +@handle_import_exceptions +def post_dependent_cost_code(import_log: ImportLog, dependent_field_setting: DependentFieldSetting, platform: PlatformConnector, filters: Dict, is_enabled: bool = True) -> List[str]: projects = CostCategory.objects.filter(**filters).values('job_name').annotate(cost_codes=ArrayAgg('cost_code_name', distinct=True)) projects_from_categories = [project['job_name'] for project in projects] posted_cost_codes = [] @@ -103,10 +106,14 @@ def post_dependent_cost_code(dependent_field_setting: DependentFieldSetting, pla logger.error(f'Exception while posting dependent cost code | Error: {exception} | Payload: {payload}') raise + import_log.status = 'COMPLETE' + import_log.error_log = [] + import_log.save() return posted_cost_codes -def post_dependent_cost_type(dependent_field_setting: DependentFieldSetting, platform: PlatformConnector, filters: Dict): +@handle_import_exceptions +def post_dependent_cost_type(import_log: ImportLog, dependent_field_setting: DependentFieldSetting, platform: PlatformConnector, filters: Dict): cost_categories = CostCategory.objects.filter(is_imported=False, **filters).values('cost_code_name').annotate(cost_categories=ArrayAgg('name', distinct=True)) for category in cost_categories: @@ -129,6 +136,10 @@ def post_dependent_cost_type(dependent_field_setting: DependentFieldSetting, pla logger.error(f'Exception while posting dependent cost type | Error: {exception} | Payload: {payload}') raise + import_log.status = 'COMPLETE' + import_log.error_log = [] + import_log.save() + def post_dependent_expense_field_values(workspace_id: int, dependent_field_setting: DependentFieldSetting, platform: PlatformConnector = None): if not platform: @@ -141,10 +152,20 @@ def post_dependent_expense_field_values(workspace_id: int, dependent_field_setti if dependent_field_setting.last_successful_import_at: filters['updated_at__gte'] = dependent_field_setting.last_successful_import_at - posted_cost_types = post_dependent_cost_code(dependent_field_setting, platform, filters) + cost_code_import_log = ImportLog.objects.filter(workspace_id=workspace_id, attribute_type='COST_CODE').first() + cost_category_import_log = ImportLog.objects.filter(workspace_id=workspace_id, attribute_type='COST_CATEGORY').first() + + posted_cost_types = post_dependent_cost_code(cost_code_import_log, dependent_field_setting, platform, filters) if posted_cost_types: filters['cost_code_name__in'] = posted_cost_types - post_dependent_cost_type(dependent_field_setting, platform, filters) + + if cost_code_import_log.status in ['FAILED', 'FATAL']: + cost_category_import_log.status = 'FAILED' + cost_category_import_log.error_log = {'message': 'Importing COST_CODE failed'} + cost_category_import_log.save() + return + else: + post_dependent_cost_type(cost_category_import_log, dependent_field_setting, platform, filters) DependentFieldSetting.objects.filter(workspace_id=workspace_id).update(last_successful_import_at=datetime.now()) diff --git a/apps/sage300/helpers.py b/apps/sage300/helpers.py index 4454d62a..4b144725 100644 --- a/apps/sage300/helpers.py +++ b/apps/sage300/helpers.py @@ -11,6 +11,7 @@ from apps.sage300.models import CostCategory from apps.fyle.models import DependentFieldSetting from apps.sage300.dependent_fields import post_dependent_cost_code +from apps.mappings.models import ImportLog logger = logging.getLogger(__name__) logger.level = logging.INFO @@ -134,9 +135,10 @@ def update_and_disable_cost_code(workspace_id: int, cost_codes_to_disable: Dict, 'job_id__in':list(cost_codes_to_disable.keys()), 'workspace_id': workspace_id } + cost_code_import_log = ImportLog.create_import_log('COST_CODE', workspace_id) # This call will disable the cost codes in Fyle that has old project name - posted_cost_codes = post_dependent_cost_code(dependent_field_setting, platform, filters, is_enabled=False) + posted_cost_codes = post_dependent_cost_code(cost_code_import_log, dependent_field_setting, platform, filters, is_enabled=False) logger.info(f"Disabled Cost Codes in Fyle | WORKSPACE_ID: {workspace_id} | COUNT: {len(posted_cost_codes)}") diff --git a/apps/sage300/utils.py b/apps/sage300/utils.py index 9d0ab55e..dfa08b25 100644 --- a/apps/sage300/utils.py +++ b/apps/sage300/utils.py @@ -5,6 +5,7 @@ from sage_desktop_sdk.sage_desktop_sdk import SageDesktopSDK from apps.sage300.models import CostCategory from apps.mappings.models import Version +from apps.mappings.exceptions import handle_import_exceptions logger = logging.getLogger(__name__) logger.level = logging.INFO @@ -262,7 +263,8 @@ def sync_commitment_items(self): ] self._sync_data(commitment_items, 'COMMITMENT_ITEM', 'commitment_item', self.workspace_id, field_names, False) - def sync_cost_codes(self): + @handle_import_exceptions + def sync_cost_codes(self, _import_log = None): """ Synchronize cost codes from Sage Desktop SDK to your application """ @@ -272,7 +274,8 @@ def sync_cost_codes(self): self._sync_data(cost_codes, 'COST_CODE', 'cost_code', self.workspace_id, field_names) return [] - def sync_cost_categories(self): + @handle_import_exceptions + def sync_cost_categories(self, import_log = None): """ Synchronize categories from Sage Desktop SDK to your application """ diff --git a/requirements.txt b/requirements.txt index b8f181c6..4c46f280 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ fyle==0.36.1 # Reusable Fyle Packages fyle-rest-auth==1.7.2 -fyle-accounting-mappings==1.33.0 +fyle-accounting-mappings==1.33.1 fyle-integrations-platform-connector==1.36.3 diff --git a/tests/test_fyle/test_views.py b/tests/test_fyle/test_views.py index 76252b73..cca4311e 100644 --- a/tests/test_fyle/test_views.py +++ b/tests/test_fyle/test_views.py @@ -10,6 +10,7 @@ def test_import_fyle_attributes(mocker, api_client, test_connection, create_temp_workspace, add_fyle_credentials): mocker.patch('fyle_integrations_platform_connector.fyle_integrations_platform_connector.PlatformConnector.import_fyle_dimensions', return_value=[]) + mocker.patch('apps.fyle.serializers.chain_import_fields_to_fyle', return_value=None) url = reverse('import-fyle-attributes', kwargs={'workspace_id': 1}) diff --git a/tests/test_sage300/test_dependent_fields.py b/tests/test_sage300/test_dependent_fields.py index f37276f6..26d482cc 100644 --- a/tests/test_sage300/test_dependent_fields.py +++ b/tests/test_sage300/test_dependent_fields.py @@ -6,6 +6,7 @@ import_dependent_fields_to_fyle ) from apps.fyle.models import DependentFieldSetting +from apps.mappings.models import ImportLog def test_construct_custom_field_placeholder(): @@ -64,14 +65,24 @@ def test_post_dependent_cost_code( } dependent_field_settings = DependentFieldSetting.objects.get(workspace_id=workspace_id) + cost_code_import_log = ImportLog.create_import_log('COST_CODE', workspace_id) result = post_dependent_cost_code( + cost_code_import_log, dependent_field_setting=dependent_field_settings, platform=platform.return_value, filters=filters ) assert result == ['Direct Mail Campaign', 'Platform APIs'] + assert cost_code_import_log.status == 'COMPLETE' + + post_dependent_cost_code( + cost_code_import_log, + dependent_field_setting=dependent_field_settings, + platform=platform.return_value + ) + assert cost_code_import_log.status == 'FATAL' def test_post_dependent_cost_type( @@ -96,13 +107,24 @@ def test_post_dependent_cost_type( dependent_field_settings = DependentFieldSetting.objects.get(workspace_id=workspace_id) + cost_category_import_log = ImportLog.create_import_log('COST_CATEGORY', workspace_id) + post_dependent_cost_type( + cost_category_import_log, dependent_field_setting=dependent_field_settings, platform=platform.return_value, filters=filters ) assert platform.return_value.dependent_fields.bulk_post_dependent_expense_field_values.call_count == 2 + assert cost_category_import_log.status == 'COMPLETE' + + post_dependent_cost_type( + cost_category_import_log, + dependent_field_setting=dependent_field_settings, + platform=platform.return_value + ) + assert cost_category_import_log.status == 'FATAL' def test_post_dependent_expense_field_values( @@ -124,6 +146,9 @@ def test_post_dependent_expense_field_values( dependent_field_settings = DependentFieldSetting.objects.get(workspace_id=workspace_id) + ImportLog.create_import_log('COST_CODE', workspace_id) + ImportLog.create_import_log('COST_CATEGORY', workspace_id) + post_dependent_expense_field_values( workspace_id=workspace_id, dependent_field_setting=dependent_field_settings @@ -150,6 +175,9 @@ def test_import_dependent_fields_to_fyle( 'dependent_fields.bulk_post_dependent_expense_field_values' ) + ImportLog.create_import_log('COST_CODE', workspace_id) + ImportLog.create_import_log('COST_CATEGORY', workspace_id) + import_dependent_fields_to_fyle(workspace_id) assert platform.return_value.dependent_fields.bulk_post_dependent_expense_field_values.call_count == 4 diff --git a/tests/test_sage300/test_utils.py b/tests/test_sage300/test_utils.py index c51c9bf9..365c0812 100644 --- a/tests/test_sage300/test_utils.py +++ b/tests/test_sage300/test_utils.py @@ -2,6 +2,7 @@ from fyle_accounting_mappings.models import DestinationAttribute from apps.mappings.models import Version from apps.workspaces.models import Workspace +from apps.mappings.models import ImportLog def test_sage_desktop_connector( @@ -453,6 +454,8 @@ def test_sync_cost_categories( workspace_id=workspace_id ) + cost_category_import_log = ImportLog.create_import_log('COST_CATEGORY', workspace_id) + mock_category = [{ "Id": 1, "JobId": "10064", @@ -480,7 +483,9 @@ def test_sync_cost_categories( sage_connector.connection.categories.get_all_categories.return_value = categories_generator - sage_connector.sync_cost_categories() + sage_connector.sync_cost_categories(cost_category_import_log) + + assert Version.objects.get(workspace_id=workspace_id).cost_category == 2 assert Version.objects.get(workspace_id=workspace_id).cost_category == 2 @@ -501,6 +506,8 @@ def test_sync_cost_codes( workspace_id=workspace_id ) + cost_code_import_log = ImportLog.create_import_log('COST_CODE', workspace_id) + Version.objects.update_or_create( workspace_id=workspace_id, cost_code=1, @@ -520,5 +527,5 @@ def test_sync_cost_codes( sage_connector.connection.cost_codes.get_all_costcodes.return_value = [[[mock_data]]] - result = sage_connector.sync_cost_codes() + result = sage_connector.sync_cost_codes(cost_code_import_log) assert result == []