From 729de0d0e8a3647d2ebb0eab0c7122b40b3b9081 Mon Sep 17 00:00:00 2001 From: labhvam5 Date: Tue, 14 Nov 2023 18:37:02 +0530 Subject: [PATCH 1/7] adding sub module changes --- .gitmodules | 3 +++ fyle-integrations-imports | 1 - fyle_integrations_imports | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .gitmodules delete mode 160000 fyle-integrations-imports create mode 160000 fyle_integrations_imports diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..dd752d7f --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "fyle_integrations_imports"] + path = fyle_integrations_imports + url = https://github.com/fylein/fyle_integrations_imports.git diff --git a/fyle-integrations-imports b/fyle-integrations-imports deleted file mode 160000 index a77d9a6f..00000000 --- a/fyle-integrations-imports +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a77d9a6f515706b55256ab0b382647702562a5c2 diff --git a/fyle_integrations_imports b/fyle_integrations_imports new file mode 160000 index 00000000..3834b930 --- /dev/null +++ b/fyle_integrations_imports @@ -0,0 +1 @@ +Subproject commit 3834b930f0db39cbaa1ea1ff2fc318e3cc4ef5e3 From 96987b7145a5c80ddb244745c0920dc47c6da122 Mon Sep 17 00:00:00 2001 From: labhvam5 Date: Tue, 14 Nov 2023 23:06:23 +0530 Subject: [PATCH 2/7] inital run changes --- Dockerfile | 2 +- apps/mappings/exceptions.py | 58 +++++++++++++++++++ apps/mappings/helpers.py | 2 +- apps/mappings/queues.py | 35 ++++++----- apps/quickbooks_online/utils.py | 6 ++ .../workspaces/apis/map_employees/triggers.py | 2 +- fyle_integrations_imports | 2 +- fyle_qbo_api/settings.py | 1 + 8 files changed, 90 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index 521b124e..06c081ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,7 @@ COPY . /fyle-qbo-api/ WORKDIR /fyle-qbo-api # Do linting checks -RUN flake8 . +# RUN flake8 . # Expose development port EXPOSE 8000 diff --git a/apps/mappings/exceptions.py b/apps/mappings/exceptions.py index dfb7e59c..2672e557 100644 --- a/apps/mappings/exceptions.py +++ b/apps/mappings/exceptions.py @@ -6,6 +6,7 @@ from qbosdk.exceptions import WrongParamsError as QBOWrongParamsError from apps.workspaces.models import QBOCredential +from fyle_integrations_imports.models import ImportLog logger = logging.getLogger(__name__) logger.level = logging.INFO @@ -50,3 +51,60 @@ def new_fn(workspace_id: int, *args): return new_fn return decorator + + +def new_handle_import_exceptions(func): + def new_fn(expense_attribute_instance, *args): + import_log: ImportLog = args[0] + workspace_id = import_log.workspace_id + attribute_type = import_log.attribute_type + error = { + 'task': 'Import {0} to Fyle and Auto Create Mappings'.format(attribute_type), + 'workspace_id': workspace_id, + 'message': None, + 'response': None + } + try: + return func(expense_attribute_instance, *args) + except WrongParamsError as exception: + error['message'] = exception.message + error['response'] = exception.response + error['alert'] = True + import_log.status = 'FAILED' + + except (QBOCredential.DoesNotExist): + error['message'] = 'Invalid Token or QBO credentials does not exist workspace_id - {0}'.format(workspace_id) + error['alert'] = False + import_log.status = 'FAILED' + + except InvalidTokenError: + error['message'] = 'Invalid Token for fyle' + error['alert'] = False + import_log.status = 'FAILED' + + except InternalServerError: + error['message'] = 'Internal server error while importing to Fyle' + error['alert'] = True + import_log.status = 'FAILED' + + except (QBOWrongParamsError, QBOInvalidTokenError) as exception: + error['message'] = 'QBO token expired' + error['alert'] = False + import_log.status = 'FAILED' + + except Exception: + response = traceback.format_exc() + error['message'] = 'Something went wrong' + error['response'] = response + error['alert'] = False + import_log.status = 'FATAL' + + if error['alert']: + logger.error(error) + else: + logger.info(error) + + import_log.error_log = error + import_log.save() + + return new_fn diff --git a/apps/mappings/helpers.py b/apps/mappings/helpers.py index 1643a8e3..b8040ab2 100644 --- a/apps/mappings/helpers.py +++ b/apps/mappings/helpers.py @@ -24,7 +24,7 @@ def get_auto_sync_permission(mapping_setting: MappingSetting): :return: bool """ is_auto_sync_status_allowed = False - if (mapping_setting.destination_field == 'PROJECT' and mapping_setting.source_field == 'CUSTOMER') or mapping_setting.source_field == 'CATEGORY': + if (mapping_setting.destination_field == 'CUSTOMER' and mapping_setting.source_field == 'PROJECT') or mapping_setting.source_field == 'CATEGORY': is_auto_sync_status_allowed = True return is_auto_sync_status_allowed diff --git a/apps/mappings/queues.py b/apps/mappings/queues.py index eb14ff75..5d69a9a4 100644 --- a/apps/mappings/queues.py +++ b/apps/mappings/queues.py @@ -7,6 +7,7 @@ from apps.mappings.models import GeneralMapping from apps.workspaces.models import WorkspaceGeneralSettings, QBOCredential from apps.mappings.helpers import get_auto_sync_permission +from fyle_integrations_imports.queues import chain_import_fields_to_fyle SYNC_METHODS = { 'ACCOUNT': 'accounts', @@ -107,26 +108,32 @@ def construct_tasks_and_chain_import_fields_to_fyle(workspace_id): mapping_settings = MappingSetting.objects.filter(workspace_id=workspace_id, import_to_fyle=True) credentials = QBOCredential.objects.get(workspace_id=workspace_id) + + print(""" + + construct_tasks_and_chain_import_fields_to_fyle + + """) + task_settings = { 'import_tax': None, 'import_vendors_as_merchants': None, 'import_categories': None, 'mapping_settings': [], - 'credentails': credentials, + 'credentials': credentials, 'sdk_connection_string': 'apps.quickbooks_online.utils.QBOConnector', } # For now we are only adding PROJECTS support that is why we are hardcoding it - for mapping_setting in mapping_settings: - if mapping_setting.source_field in ['PROJECT']: - task_settings['mapping_settings'].append({ - 'source_field': mapping_setting.source_field, - 'destination_field': mapping_setting.destination_field, - 'destination_sync_method': SYNC_METHODS[mapping_setting.destination_field], - 'is_auto_sync_enabled': get_auto_sync_permission(mapping_setting), - 'is_custom': False, - }) - - # Make a call to the SDK - # chain_import_fields_to_fyle(workspace_id, task_settings) - pass + if mapping_settings: + for mapping_setting in mapping_settings: + if mapping_setting.source_field in ['PROJECT']: + task_settings['mapping_settings'].append({ + 'source_field': mapping_setting.source_field, + 'destination_field': mapping_setting.destination_field, + 'destination_sync_method': SYNC_METHODS[mapping_setting.destination_field], + 'is_auto_sync_enabled': get_auto_sync_permission(mapping_setting), + 'is_custom': False, + }) + + chain_import_fields_to_fyle(workspace_id, task_settings) diff --git a/apps/quickbooks_online/utils.py b/apps/quickbooks_online/utils.py index 12aa9d4f..b58244ae 100644 --- a/apps/quickbooks_online/utils.py +++ b/apps/quickbooks_online/utils.py @@ -374,6 +374,12 @@ def sync_customers(self): """ Get customers """ + print(""" + + + sync_customers + + """) customers_count = self.connection.customers.count() if customers_count < SYNC_UPPER_LIMIT['customers']: customers = self.connection.customers.get() diff --git a/apps/workspaces/apis/map_employees/triggers.py b/apps/workspaces/apis/map_employees/triggers.py index 882041d5..c658b60f 100644 --- a/apps/workspaces/apis/map_employees/triggers.py +++ b/apps/workspaces/apis/map_employees/triggers.py @@ -1,4 +1,4 @@ -from apps.mappings.queue import schedule_auto_map_employees +from apps.mappings.queues import schedule_auto_map_employees from apps.workspaces.models import WorkspaceGeneralSettings diff --git a/fyle_integrations_imports b/fyle_integrations_imports index 3834b930..18f4fbd0 160000 --- a/fyle_integrations_imports +++ b/fyle_integrations_imports @@ -1 +1 @@ -Subproject commit 3834b930f0db39cbaa1ea1ff2fc318e3cc4ef5e3 +Subproject commit 18f4fbd0ad156fc4777307031d9f01bfa26e1678 diff --git a/fyle_qbo_api/settings.py b/fyle_qbo_api/settings.py index 970e9477..05d7b262 100644 --- a/fyle_qbo_api/settings.py +++ b/fyle_qbo_api/settings.py @@ -44,6 +44,7 @@ 'corsheaders', 'fyle_rest_auth', 'fyle_accounting_mappings', + 'fyle_integrations_imports', 'django_q', 'django_filters', # User Created Apps From 1886cfcf4b8cf0901e6ecaf1c1f275a85791ae4d Mon Sep 17 00:00:00 2001 From: labhvam5 Date: Wed, 15 Nov 2023 20:37:55 +0530 Subject: [PATCH 3/7] adding is_auto_sync_enabled as project property --- fyle_integrations_imports | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fyle_integrations_imports b/fyle_integrations_imports index 18f4fbd0..e2881da7 160000 --- a/fyle_integrations_imports +++ b/fyle_integrations_imports @@ -1 +1 @@ -Subproject commit 18f4fbd0ad156fc4777307031d9f01bfa26e1678 +Subproject commit e2881da7eddcb9d69834c121bbc69d87d18d3c8c From dcbe212eab317dde3d7491798ebf9e17079f0f57 Mon Sep 17 00:00:00 2001 From: labhvam5 Date: Thu, 16 Nov 2023 18:44:11 +0530 Subject: [PATCH 4/7] minor changes --- apps/mappings/queues.py | 3 ++- fyle_integrations_imports | 2 +- sql/functions/delete-workspace.sql | 6 ++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/apps/mappings/queues.py b/apps/mappings/queues.py index 5d69a9a4..474d0970 100644 --- a/apps/mappings/queues.py +++ b/apps/mappings/queues.py @@ -8,6 +8,7 @@ from apps.workspaces.models import WorkspaceGeneralSettings, QBOCredential from apps.mappings.helpers import get_auto_sync_permission from fyle_integrations_imports.queues import chain_import_fields_to_fyle +from fyle_integrations_imports.helpers import TaskSettings SYNC_METHODS = { 'ACCOUNT': 'accounts', @@ -115,7 +116,7 @@ def construct_tasks_and_chain_import_fields_to_fyle(workspace_id): """) - task_settings = { + task_settings: TaskSettings = { 'import_tax': None, 'import_vendors_as_merchants': None, 'import_categories': None, diff --git a/fyle_integrations_imports b/fyle_integrations_imports index e2881da7..5f887920 160000 --- a/fyle_integrations_imports +++ b/fyle_integrations_imports @@ -1 +1 @@ -Subproject commit e2881da7eddcb9d69834c121bbc69d87d18d3c8c +Subproject commit 5f8879202cb2ad62bbd6689aa2aa24f2ed5fc683 diff --git a/sql/functions/delete-workspace.sql b/sql/functions/delete-workspace.sql index 175206f6..87c40fbb 100644 --- a/sql/functions/delete-workspace.sql +++ b/sql/functions/delete-workspace.sql @@ -25,6 +25,12 @@ BEGIN GET DIAGNOSTICS rcount = ROW_COUNT; RAISE NOTICE 'Deleted % errors', rcount; + DELETE + FROM import_logs il + WHERE il.workspace_id = _workspace_id; + GET DIAGNOSTICS rcount = ROW_COUNT; + RAISE NOTICE 'Deleted % import_logs', rcount; + DELETE FROM bill_lineitems bl WHERE bl.bill_id IN ( From 2601946926ad963beceb7c4b7f8117b853b3b4d0 Mon Sep 17 00:00:00 2001 From: labhvam5 Date: Thu, 16 Nov 2023 18:52:05 +0530 Subject: [PATCH 5/7] removing print statements --- Dockerfile | 2 +- apps/mappings/queues.py | 7 ------- apps/quickbooks_online/utils.py | 6 ------ fyle_integrations_imports | 2 +- 4 files changed, 2 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 06c081ad..521b124e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,7 @@ COPY . /fyle-qbo-api/ WORKDIR /fyle-qbo-api # Do linting checks -# RUN flake8 . +RUN flake8 . # Expose development port EXPOSE 8000 diff --git a/apps/mappings/queues.py b/apps/mappings/queues.py index 474d0970..720ec1ff 100644 --- a/apps/mappings/queues.py +++ b/apps/mappings/queues.py @@ -109,13 +109,6 @@ def construct_tasks_and_chain_import_fields_to_fyle(workspace_id): mapping_settings = MappingSetting.objects.filter(workspace_id=workspace_id, import_to_fyle=True) credentials = QBOCredential.objects.get(workspace_id=workspace_id) - - print(""" - - construct_tasks_and_chain_import_fields_to_fyle - - """) - task_settings: TaskSettings = { 'import_tax': None, 'import_vendors_as_merchants': None, diff --git a/apps/quickbooks_online/utils.py b/apps/quickbooks_online/utils.py index b58244ae..12aa9d4f 100644 --- a/apps/quickbooks_online/utils.py +++ b/apps/quickbooks_online/utils.py @@ -374,12 +374,6 @@ def sync_customers(self): """ Get customers """ - print(""" - - - sync_customers - - """) customers_count = self.connection.customers.count() if customers_count < SYNC_UPPER_LIMIT['customers']: customers = self.connection.customers.get() diff --git a/fyle_integrations_imports b/fyle_integrations_imports index 5f887920..d82c3be8 160000 --- a/fyle_integrations_imports +++ b/fyle_integrations_imports @@ -1 +1 @@ -Subproject commit 5f8879202cb2ad62bbd6689aa2aa24f2ed5fc683 +Subproject commit d82c3be859709e754070cfbd515fbb3a15025c97 From e03eb3a03a3e0a28decfe32ada98d614800e0ce6 Mon Sep 17 00:00:00 2001 From: labhvam5 Date: Fri, 17 Nov 2023 15:26:05 +0530 Subject: [PATCH 6/7] resolving comment for sub module --- apps/mappings/exceptions.py | 2 +- apps/mappings/queues.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/mappings/exceptions.py b/apps/mappings/exceptions.py index 2672e557..976d9258 100644 --- a/apps/mappings/exceptions.py +++ b/apps/mappings/exceptions.py @@ -53,7 +53,7 @@ def new_fn(workspace_id: int, *args): return decorator -def new_handle_import_exceptions(func): +def handle_import_exceptions_v2(func): def new_fn(expense_attribute_instance, *args): import_log: ImportLog = args[0] workspace_id = import_log.workspace_id diff --git a/apps/mappings/queues.py b/apps/mappings/queues.py index 720ec1ff..3a339af0 100644 --- a/apps/mappings/queues.py +++ b/apps/mappings/queues.py @@ -8,7 +8,7 @@ from apps.workspaces.models import WorkspaceGeneralSettings, QBOCredential from apps.mappings.helpers import get_auto_sync_permission from fyle_integrations_imports.queues import chain_import_fields_to_fyle -from fyle_integrations_imports.helpers import TaskSettings +from fyle_integrations_imports.dataclasses import TasksSettings SYNC_METHODS = { 'ACCOUNT': 'accounts', @@ -109,7 +109,7 @@ def construct_tasks_and_chain_import_fields_to_fyle(workspace_id): mapping_settings = MappingSetting.objects.filter(workspace_id=workspace_id, import_to_fyle=True) credentials = QBOCredential.objects.get(workspace_id=workspace_id) - task_settings: TaskSettings = { + task_settings: TasksSettings = { 'import_tax': None, 'import_vendors_as_merchants': None, 'import_categories': None, From 07a14924d7e2a6b5726959febd8c5bd8ab37461b Mon Sep 17 00:00:00 2001 From: labhvam5 Date: Fri, 17 Nov 2023 16:10:41 +0530 Subject: [PATCH 7/7] resolving comments --- apps/mappings/exceptions.py | 9 ++------- apps/mappings/queues.py | 4 ++-- fyle_integrations_imports | 2 +- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/apps/mappings/exceptions.py b/apps/mappings/exceptions.py index 976d9258..01512144 100644 --- a/apps/mappings/exceptions.py +++ b/apps/mappings/exceptions.py @@ -72,11 +72,6 @@ def new_fn(expense_attribute_instance, *args): error['alert'] = True import_log.status = 'FAILED' - except (QBOCredential.DoesNotExist): - error['message'] = 'Invalid Token or QBO credentials does not exist workspace_id - {0}'.format(workspace_id) - error['alert'] = False - import_log.status = 'FAILED' - except InvalidTokenError: error['message'] = 'Invalid Token for fyle' error['alert'] = False @@ -87,8 +82,8 @@ def new_fn(expense_attribute_instance, *args): error['alert'] = True import_log.status = 'FAILED' - except (QBOWrongParamsError, QBOInvalidTokenError) as exception: - error['message'] = 'QBO token expired' + except (QBOWrongParamsError, QBOInvalidTokenError, QBOCredential.DoesNotExist) as exception: + error['message'] = 'Invalid Token or QBO credentials does not exist workspace_id - {0}'.format(workspace_id) error['alert'] = False import_log.status = 'FAILED' diff --git a/apps/mappings/queues.py b/apps/mappings/queues.py index 3a339af0..de5a30b0 100644 --- a/apps/mappings/queues.py +++ b/apps/mappings/queues.py @@ -8,7 +8,7 @@ from apps.workspaces.models import WorkspaceGeneralSettings, QBOCredential from apps.mappings.helpers import get_auto_sync_permission from fyle_integrations_imports.queues import chain_import_fields_to_fyle -from fyle_integrations_imports.dataclasses import TasksSettings +from fyle_integrations_imports.dataclasses import TaskSetting SYNC_METHODS = { 'ACCOUNT': 'accounts', @@ -109,7 +109,7 @@ def construct_tasks_and_chain_import_fields_to_fyle(workspace_id): mapping_settings = MappingSetting.objects.filter(workspace_id=workspace_id, import_to_fyle=True) credentials = QBOCredential.objects.get(workspace_id=workspace_id) - task_settings: TasksSettings = { + task_settings: TaskSetting = { 'import_tax': None, 'import_vendors_as_merchants': None, 'import_categories': None, diff --git a/fyle_integrations_imports b/fyle_integrations_imports index d82c3be8..5be2be9f 160000 --- a/fyle_integrations_imports +++ b/fyle_integrations_imports @@ -1 +1 @@ -Subproject commit d82c3be859709e754070cfbd515fbb3a15025c97 +Subproject commit 5be2be9f504f0bfbaebb47b2b37cf97bafcaf96d