import-projects-rework (#515)

* Adding scheduling logic

* lint fix

* lint fix

* fixing comments

* import-projects-base

* Import projects base imp (#519)

* adding sub module  changes

* inital run changes

* adding is_auto_sync_enabled as project property

* minor changes

* removing print statements

* resolving comment for sub module

* resolving comments

* lint fixes

* switching to master branch

* adding app label

* removing old code (#517)

* removing old code

* removing old test and lint fix

* Adding projects test (#522)

* Adding projects test

* adding new test

* addin exceptions test

* lint fix for test

* lint fixes

* fyle_integrations_import latest master

* adding schedule script

* lint fix

* rm sub module

* add submodule

* add submodule fetch

---------

Co-authored-by: ashwin1111 <[email protected]>

.github/workflows/codecov.yml

@@ -13,6 +13,8 @@ jobs:
     environment: CI Environment
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - uses: satackey/[email protected]
         continue-on-error: true
       - name: Bring up Services and Run Tests

.github/workflows/production_deployment.yml

@@ -10,6 +10,8 @@ jobs:
     environment: Production
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - uses: satackey/[email protected]
         continue-on-error: true
       - name: push to dockerhub

.github/workflows/pytest.yml

@@ -10,6 +10,8 @@ jobs:
     environment: CI Environment
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Bring up Services and test for token health
         run: |
           docker-compose -f docker-compose-pipeline.yml build

.github/workflows/staging_deployment.yml

@@ -14,6 +14,8 @@ jobs:
     environment: Staging
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - uses: satackey/[email protected]
         continue-on-error: true
       - name: push to dockerhub

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "fyle_integrations_imports"]
+	path = fyle_integrations_imports
+	url = https://github.com/fylein/fyle_integrations_imports/

apps/mappings/exceptions.py

@@ -6,6 +6,7 @@
 from qbosdk.exceptions import WrongParamsError as QBOWrongParamsError
 
 from apps.workspaces.models import QBOCredential
+from fyle_integrations_imports.models import ImportLog
 
 logger = logging.getLogger(__name__)
 logger.level = logging.INFO
@@ -50,3 +51,56 @@ def new_fn(workspace_id: int, *args):
         return new_fn
 
     return decorator
+
+
+def handle_import_exceptions_v2(func):
+    def new_fn(expense_attribute_instance, *args):
+        import_log: ImportLog = args[0]
+        workspace_id = import_log.workspace_id
+        attribute_type = import_log.attribute_type
+        error = {
+            'task': 'Import {0} to Fyle and Auto Create Mappings'.format(attribute_type),
+            'workspace_id': workspace_id,
+            'message': None,
+            'response': None
+        }
+        try:
+            return func(expense_attribute_instance, *args)
+        except WrongParamsError as exception:
+            error['message'] = exception.message
+            error['response'] = exception.response
+            error['alert'] = True
+            import_log.status = 'FAILED'
+
+        except InvalidTokenError:
+            error['message'] = 'Invalid Token for fyle'
+            error['alert'] = False
+            import_log.status = 'FAILED'
+
+        except InternalServerError:
+            error['message'] = 'Internal server error while importing to Fyle'
+            error['alert'] = True
+            import_log.status = 'FAILED'
+
+        except (QBOWrongParamsError, QBOInvalidTokenError, QBOCredential.DoesNotExist) as exception:
+            error['message'] = 'Invalid Token or QBO credentials does not exist workspace_id - {0}'.format(workspace_id)
+            error['alert'] = False
+            error['response'] = exception.__dict__
+            import_log.status = 'FAILED'
+
+        except Exception:
+            response = traceback.format_exc()
+            error['message'] = 'Something went wrong'
+            error['response'] = response
+            error['alert'] = False
+            import_log.status = 'FATAL'
+
+        if error['alert']:
+            logger.error(error)
+        else:
+            logger.info(error)
+
+        import_log.error_log = error
+        import_log.save()
+
+    return new_fn

apps/mappings/helpers.py

@@ -11,9 +11,20 @@ def schedule_or_delete_fyle_import_tasks(configuration: WorkspaceGeneralSettings
     :param configuration: WorkspaceGeneralSettings Instance
     :return: None
     """
-    project_mapping = MappingSetting.objects.filter(source_field='PROJECT', workspace_id=configuration.workspace_id).first()
-    if configuration.import_categories or configuration.import_items or (project_mapping and project_mapping.import_to_fyle) or configuration.import_vendors_as_merchants:
+    if configuration.import_categories or configuration.import_items or configuration.import_vendors_as_merchants:
         start_datetime = datetime.now()
         Schedule.objects.update_or_create(func='apps.mappings.tasks.auto_import_and_map_fyle_fields', args='{}'.format(configuration.workspace_id), defaults={'schedule_type': Schedule.MINUTES, 'minutes': 24 * 60, 'next_run': start_datetime})
-    elif not configuration.import_categories and not configuration.import_items and not (project_mapping and project_mapping.import_to_fyle) and not configuration.import_vendors_as_merchants:
+    elif not configuration.import_categories and not configuration.import_items and not configuration.import_vendors_as_merchants:
         Schedule.objects.filter(func='apps.mappings.tasks.auto_import_and_map_fyle_fields', args='{}'.format(configuration.workspace_id)).delete()
+
+
+def get_auto_sync_permission(mapping_setting: MappingSetting):
+    """
+    Get the auto sync permission
+    :return: bool
+    """
+    is_auto_sync_status_allowed = False
+    if (mapping_setting.destination_field == 'CUSTOMER' and mapping_setting.source_field == 'PROJECT') or mapping_setting.source_field == 'CATEGORY':
+        is_auto_sync_status_allowed = True
+
+    return is_auto_sync_status_allowed

apps/mappings/queue.py → apps/mappings/queues.py

@@ -5,7 +5,20 @@
 from fyle_accounting_mappings.models import MappingSetting
 
 from apps.mappings.models import GeneralMapping
-from apps.workspaces.models import WorkspaceGeneralSettings
+from apps.workspaces.models import WorkspaceGeneralSettings, QBOCredential
+from apps.mappings.helpers import get_auto_sync_permission
+from fyle_integrations_imports.queues import chain_import_fields_to_fyle
+from fyle_integrations_imports.dataclasses import TaskSetting
+
+SYNC_METHODS = {
+    'ACCOUNT': 'accounts',
+    'ITEM': 'items',
+    'VENDOR': 'vendors',
+    'DEPARTMENT': 'departments',
+    'TAX_CODE': 'tax_codes',
+    'CLASS': 'classes',
+    'CUSTOMER': 'customers',
+}
 
 
 def async_auto_create_expense_field_mapping(mapping_setting: MappingSetting):
@@ -86,3 +99,35 @@ def schedule_auto_map_employees(employee_mapping_preference: str, workspace_id:
 
 def async_disable_category_for_items_mapping(workspace_id: int):
     async_task('apps.mappings.tasks.disable_category_for_items_mapping', workspace_id)
+
+
+def construct_tasks_and_chain_import_fields_to_fyle(workspace_id):
+    """
+    Chain import fields to Fyle
+    :param workspace_id: Workspace Id
+    """
+    mapping_settings = MappingSetting.objects.filter(workspace_id=workspace_id, import_to_fyle=True)
+    credentials = QBOCredential.objects.get(workspace_id=workspace_id)
+
+    task_settings: TaskSetting = {
+        'import_tax': None,
+        'import_vendors_as_merchants': None,
+        'import_categories': None,
+        'mapping_settings': [],
+        'credentials': credentials,
+        'sdk_connection_string': 'apps.quickbooks_online.utils.QBOConnector',
+    }
+
+    # For now we are only adding PROJECTS support that is why we are hardcoding it
+    if mapping_settings:
+        for mapping_setting in mapping_settings:
+            if mapping_setting.source_field in ['PROJECT']:
+                task_settings['mapping_settings'].append({
+                    'source_field': mapping_setting.source_field,
+                    'destination_field': mapping_setting.destination_field,
+                    'destination_sync_method': SYNC_METHODS[mapping_setting.destination_field],
+                    'is_auto_sync_enabled': get_auto_sync_permission(mapping_setting),
+                    'is_custom': False,
+                })
+
+    chain_import_fields_to_fyle(workspace_id, task_settings)

apps/mappings/schedules.py

@@ -0,0 +1,42 @@
+from datetime import datetime
+from django_q.models import Schedule
+from apps.workspaces.models import WorkspaceGeneralSettings
+from fyle_accounting_mappings.models import MappingSetting
+
+
+def schedule_or_delete_fyle_import_tasks(workspace_general_settings: WorkspaceGeneralSettings, mapping_setting_instance: MappingSetting = None):
+    """
+    Schedule or delete Fyle import tasks based on the configuration.
+    :param configuration: Workspace Configuration Instance
+    :param instance: Mapping Setting Instance
+    :return: None
+    """
+    task_to_be_scheduled = None
+    # Check if there is a task to be scheduled
+    if mapping_setting_instance and mapping_setting_instance.import_to_fyle:
+        task_to_be_scheduled = mapping_setting_instance
+
+    if task_to_be_scheduled:
+        Schedule.objects.update_or_create(
+            func='apps.mappings.queues.construct_tasks_and_chain_import_fields_to_fyle',
+            args='{}'.format(workspace_general_settings.workspace_id),
+            defaults={
+                'schedule_type': Schedule.MINUTES,
+                'minutes': 24 * 60,
+                'next_run': datetime.now()
+            }
+        )
+        return
+
+    import_fields_count = MappingSetting.objects.filter(
+        import_to_fyle=True,
+        workspace_id=workspace_general_settings.workspace_id,
+        source_field__in=['PROJECT']
+    ).count()
+
+    # If the import fields count is 0, delete the schedule
+    if import_fields_count == 0:
+        Schedule.objects.filter(
+            func='apps.mappings.queues.construct_tasks_and_chain_import_fields_to_fyle',
+            args='{}'.format(workspace_general_settings.workspace_id)
+        ).delete()

apps/mappings/signals.py

@@ -5,13 +5,14 @@
 from django.dispatch import receiver
 from fyle_accounting_mappings.models import EmployeeMapping, Mapping, MappingSetting
 
-from apps.mappings.helpers import schedule_or_delete_fyle_import_tasks
-from apps.mappings.queue import (
+from apps.mappings.queues import (
     async_auto_create_expense_field_mapping,
     schedule_cost_centers_creation,
     schedule_fyle_attributes_creation,
 )
 from apps.mappings.tasks import upload_attributes_to_fyle
+# TODO: Fix the naming convention when we remove the old schedule_or_delete_fyle_import_tasks import from helpers.py
+from apps.mappings.schedules import schedule_or_delete_fyle_import_tasks as new_schedule_or_delete_fyle_import_tasks
 from apps.tasks.models import Error
 from apps.workspaces.apis.import_settings.triggers import ImportSettingsTrigger
 from apps.workspaces.models import WorkspaceGeneralSettings
@@ -51,7 +52,7 @@ def run_post_mapping_settings_triggers(sender, instance: MappingSetting, **kwarg
     workspace_general_settings = WorkspaceGeneralSettings.objects.filter(workspace_id=instance.workspace_id).first()
 
     if instance.source_field == 'PROJECT':
-        schedule_or_delete_fyle_import_tasks(workspace_general_settings)
+        new_schedule_or_delete_fyle_import_tasks(workspace_general_settings, instance)
 
     if instance.source_field == 'COST_CENTER':
         schedule_cost_centers_creation(instance.import_to_fyle, int(instance.workspace_id))

apps/mappings/tasks.py

@@ -142,54 +142,6 @@ def disable_or_enable_expense_attributes(source_field: str, destination_field: s
         return expense_attributes_ids
 
 
-def create_fyle_projects_payload(projects: List[DestinationAttribute], existing_project_names: list, updated_projects: List[ExpenseAttribute] = None):
-    """
-    Create Fyle Projects Payload from QBO Customer / Projects
-    :param projects: QBO Projects
-    :param existing_project_names: Existing Projects in Fyle
-    :return: Fyle Projects Payload
-    """
-    payload = []
-    existing_project_names = [project_name.lower() for project_name in existing_project_names]
-    if updated_projects:
-        for project in updated_projects:
-            destination_id_of_project = project.mapping.first().destination.destination_id
-            payload.append({'id': project.source_id, 'name': project.value, 'code': destination_id_of_project, 'description': 'Project - {0}, Id - {1}'.format(project.value, destination_id_of_project), 'is_enabled': project.active})
-    else:
-        for project in projects:
-            if project.value.lower() not in existing_project_names:
-                payload.append({'name': project.value, 'code': project.destination_id, 'description': 'Project - {0}, Id - {1}'.format(project.value, project.destination_id), 'is_enabled': project.active})
-
-    return payload
-
-
-def post_projects_in_batches(platform: PlatformConnector, workspace_id: int, destination_field: str):
-    existing_project_names = ExpenseAttribute.objects.filter(attribute_type='PROJECT', workspace_id=workspace_id).values_list('value', flat=True)
-    qbo_attributes_count = DestinationAttribute.objects.filter(attribute_type=destination_field, workspace_id=workspace_id).count()
-
-    page_size = 200
-    for offset in range(0, qbo_attributes_count, page_size):
-        limit = offset + page_size
-        paginated_qbo_attributes = DestinationAttribute.objects.filter(attribute_type=destination_field, workspace_id=workspace_id).order_by('value', 'id')[offset:limit]
-
-        paginated_qbo_attributes = remove_duplicates(paginated_qbo_attributes)
-
-        fyle_payload: List[Dict] = create_fyle_projects_payload(paginated_qbo_attributes, existing_project_names)
-        if fyle_payload:
-            platform.projects.post_bulk(fyle_payload)
-            platform.projects.sync()
-
-        Mapping.bulk_create_mappings(paginated_qbo_attributes, 'PROJECT', destination_field, workspace_id)
-
-    if destination_field == 'CUSTOMER':
-        project_ids_to_be_changed = disable_or_enable_expense_attributes('PROJECT', 'CUSTOMER', workspace_id)
-        if project_ids_to_be_changed:
-            expense_attributes = ExpenseAttribute.objects.filter(id__in=project_ids_to_be_changed)
-            fyle_payload: List[Dict] = create_fyle_projects_payload(projects=[], existing_project_names=[], updated_projects=expense_attributes)
-            platform.projects.post_bulk(fyle_payload)
-            platform.projects.sync()
-
-
 @handle_import_exceptions(task_name='Auto Create Tax Code Mappings')
 def auto_create_tax_codes_mappings(workspace_id: int):
     """
@@ -209,24 +161,6 @@ def auto_create_tax_codes_mappings(workspace_id: int):
     upload_tax_groups_to_fyle(platform, workspace_id)
 
 
-@handle_import_exceptions(task_name='Auto Create Project Mappings')
-def auto_create_project_mappings(workspace_id: int):
-    """
-    Create Project Mappings
-    :return: mappings
-    """
-    fyle_credentials: FyleCredential = FyleCredential.objects.get(workspace_id=workspace_id)
-    platform = PlatformConnector(fyle_credentials)
-
-    platform.projects.sync()
-
-    mapping_setting = MappingSetting.objects.get(source_field='PROJECT', workspace_id=workspace_id)
-
-    sync_qbo_attribute(mapping_setting.destination_field, workspace_id)
-
-    post_projects_in_batches(platform, workspace_id, mapping_setting.destination_field)
-
-
 def create_fyle_categories_payload(categories: List[DestinationAttribute], workspace_id: int, updated_categories: List[ExpenseAttribute] = None):
     """
     Create Fyle Categories Payload from QBO Customer / Categories
@@ -792,7 +726,6 @@ def auto_import_and_map_fyle_fields(workspace_id):
     Auto import and map fyle fields
     """
     workspace_general_settings: WorkspaceGeneralSettings = WorkspaceGeneralSettings.objects.get(workspace_id=workspace_id)
-    project_mapping = MappingSetting.objects.filter(source_field='PROJECT', workspace_id=workspace_general_settings.workspace_id).first()
 
     chain = Chain()
 
@@ -802,8 +735,5 @@ def auto_import_and_map_fyle_fields(workspace_id):
     if workspace_general_settings.import_categories or workspace_general_settings.import_items:
         chain.append('apps.mappings.tasks.auto_create_category_mappings', workspace_id)
 
-    if project_mapping and project_mapping.import_to_fyle:
-        chain.append('apps.mappings.tasks.auto_create_project_mappings', workspace_id)
-
     if chain.length() > 0:
         chain.run()

apps/workspaces/apis/advanced_configurations/triggers.py

@@ -1,6 +1,6 @@
 import logging
 
-from apps.mappings.queue import schedule_bill_payment_creation
+from apps.mappings.queues import schedule_bill_payment_creation
 from apps.quickbooks_online.queue import schedule_qbo_objects_status_sync, schedule_reimbursements_sync
 from apps.workspaces.models import WorkspaceGeneralSettings
 from apps.workspaces.actions import post_to_integration_settings

apps/workspaces/apis/export_settings/triggers.py

@@ -1,6 +1,6 @@
 from typing import Dict
 
-from apps.mappings.queue import async_disable_category_for_items_mapping
+from apps.mappings.queues import async_disable_category_for_items_mapping
 from apps.workspaces.models import WorkspaceGeneralSettings
 
 

apps/workspaces/apis/import_settings/triggers.py

@@ -5,7 +5,7 @@
 
 from apps.fyle.models import ExpenseGroupSettings
 from apps.mappings.helpers import schedule_or_delete_fyle_import_tasks
-from apps.mappings.queue import (
+from apps.mappings.queues import (
     async_disable_category_for_items_mapping,
     schedule_cost_centers_creation,
     schedule_fyle_attributes_creation,

apps/workspaces/apis/map_employees/triggers.py

@@ -1,4 +1,4 @@
-from apps.mappings.queue import schedule_auto_map_employees
+from apps.mappings.queues import schedule_auto_map_employees
 from apps.workspaces.models import WorkspaceGeneralSettings
 
 

apps/workspaces/utils.py

@@ -9,7 +9,7 @@
 from qbosdk import InternalServerError, NotFoundClientError, UnauthorizedClientError, WrongParamsError
 
 from apps.fyle.models import ExpenseGroupSettings
-from apps.mappings.queue import (
+from apps.mappings.queues import (
     schedule_auto_map_ccc_employees,
     schedule_auto_map_employees,
     schedule_bill_payment_creation,