ExpenseCustomField Rework

apps/mappings/constants.py

@@ -43,3 +43,13 @@
     'paid date',
     'expense created date',
 ]
+
+SYNC_METHODS = {
+    'ACCOUNT': 'accounts',
+    'ITEM': 'items',
+    'VENDOR': 'vendors',
+    'DEPARTMENT': 'departments',
+    'TAX_CODE': 'tax_codes',
+    'CLASS': 'classes',
+    'CUSTOMER': 'customers',
+}

apps/mappings/queues.py

@@ -1,41 +1,14 @@
-from datetime import datetime, timedelta
+from datetime import datetime
 
 from django_q.models import Schedule
-from django_q.tasks import async_task
 from fyle_accounting_mappings.models import MappingSetting
 
 from apps.mappings.models import GeneralMapping
 from apps.workspaces.models import WorkspaceGeneralSettings, QBOCredential
 from apps.mappings.helpers import get_auto_sync_permission
 from fyle_integrations_imports.queues import chain_import_fields_to_fyle
 from fyle_integrations_imports.dataclasses import TaskSetting
-
-SYNC_METHODS = {
-    'ACCOUNT': 'accounts',
-    'ITEM': 'items',
-    'VENDOR': 'vendors',
-    'DEPARTMENT': 'departments',
-    'TAX_CODE': 'tax_codes',
-    'CLASS': 'classes',
-    'CUSTOMER': 'customers',
-}
-
-
-def async_auto_create_expense_field_mapping(mapping_setting: MappingSetting):
-    async_task('apps.mappings.tasks.auto_create_expense_fields_mappings', int(mapping_setting.workspace_id), mapping_setting.destination_field, mapping_setting.source_field)
-
-
-def schedule_fyle_attributes_creation(workspace_id: int):
-    mapping_settings = MappingSetting.objects.filter(is_custom=True, import_to_fyle=True, workspace_id=workspace_id).all()
-    if mapping_settings:
-        schedule, _ = Schedule.objects.get_or_create(
-            func='apps.mappings.tasks.async_auto_create_custom_field_mappings', args='{0}'.format(workspace_id), defaults={'schedule_type': Schedule.MINUTES, 'minutes': 24 * 60, 'next_run': datetime.now() + timedelta(hours=24)}
-        )
-    else:
-        schedule: Schedule = Schedule.objects.filter(func='apps.mappings.tasks.async_auto_create_custom_field_mappings', args='{0}'.format(workspace_id)).first()
-
-        if schedule:
-            schedule.delete()
+from apps.mappings.constants import SYNC_METHODS
 
 
 def schedule_bill_payment_creation(sync_fyle_to_qbo_payments, workspace_id):
@@ -127,13 +100,13 @@ def construct_tasks_and_chain_import_fields_to_fyle(workspace_id):
     # For now we are only adding PROJECTS support that is why we are hardcoding it
     if mapping_settings:
         for mapping_setting in mapping_settings:
-            if mapping_setting.source_field in ['PROJECT', 'COST_CENTER']:
+            if mapping_setting.source_field in ['PROJECT', 'COST_CENTER'] or mapping_setting.is_custom:
                 task_settings['mapping_settings'].append({
                     'source_field': mapping_setting.source_field,
                     'destination_field': mapping_setting.destination_field,
                     'destination_sync_methods': [SYNC_METHODS[mapping_setting.destination_field]],
                     'is_auto_sync_enabled': get_auto_sync_permission(workspace_general_settings, mapping_setting),
-                    'is_custom': False,
+                    'is_custom': mapping_setting.is_custom
                 })
 
     chain_import_fields_to_fyle(workspace_id, task_settings)

apps/mappings/schedules.py

@@ -1,20 +1,22 @@
 from datetime import datetime
+from typing import Dict, List
 from django_q.models import Schedule
 from apps.workspaces.models import WorkspaceGeneralSettings
 from fyle_accounting_mappings.models import MappingSetting
 
 
-def schedule_or_delete_fyle_import_tasks(workspace_general_settings: WorkspaceGeneralSettings, mapping_setting_instance: MappingSetting = None):
+def schedule_or_delete_fyle_import_tasks(workspace_general_settings: WorkspaceGeneralSettings, mapping_settings: List[Dict] = []):
     """
     Schedule or delete Fyle import tasks based on the workspace_general_settings and mapping_settings.
     :param workspace_general_settings: Workspace workspace_general_settings Instance
     :param instance: Mapping Setting Instance
     :return: None
     """
     task_to_be_scheduled = None
-    # Check if there is a task to be scheduled
-    if mapping_setting_instance and mapping_setting_instance.import_to_fyle:
-        task_to_be_scheduled = mapping_setting_instance
+    for mapping_setting in mapping_settings:
+        if mapping_setting['import_to_fyle'] and mapping_setting['source_field'] in ['PROJECT', 'COST_CENTER'] or mapping_setting['is_custom']:
+            task_to_be_scheduled = True
+            break
 
     if task_to_be_scheduled or workspace_general_settings.import_categories or workspace_general_settings.import_items:
         Schedule.objects.update_or_create(
@@ -26,17 +28,24 @@ def schedule_or_delete_fyle_import_tasks(workspace_general_settings: WorkspaceGe
                 'next_run': datetime.now()
             }
         )
-        return
 
-    import_fields_count = MappingSetting.objects.filter(
-        import_to_fyle=True,
-        workspace_id=workspace_general_settings.workspace_id,
-        source_field__in=['PROJECT', 'COST_CENTER']
-    ).count()
+    else:
+        import_fields_count = MappingSetting.objects.filter(
+            import_to_fyle=True,
+            workspace_id=workspace_general_settings.workspace_id,
+            source_field__in=['PROJECT', 'COST_CENTER']
+        ).count()
 
-    # If the import fields count is 0, delete the schedule
-    if import_fields_count == 0 and not workspace_general_settings.import_categories and not workspace_general_settings.import_items:
-        Schedule.objects.filter(
-            func='apps.mappings.queues.construct_tasks_and_chain_import_fields_to_fyle',
-            args='{}'.format(workspace_general_settings.workspace_id)
-        ).delete()
+        custom_field_import_fields_count = MappingSetting.objects.filter(
+            import_to_fyle=True,
+            workspace_id=workspace_general_settings.workspace_id,
+            is_custom=True
+        ).count()
+
+        # If the import fields count is 0, delete the schedule
+        if import_fields_count == 0 and custom_field_import_fields_count == 0\
+            and not workspace_general_settings.import_categories and not workspace_general_settings.import_items:
+            Schedule.objects.filter(
+                func='apps.mappings.queues.construct_tasks_and_chain_import_fields_to_fyle',
+                args='{}'.format(workspace_general_settings.workspace_id)
+            ).delete()

apps/mappings/signals.py

@@ -1,21 +1,24 @@
 """
 Mapping Signals
 """
+import logging
 from django.db.models.signals import post_delete, post_save, pre_save
 from django.dispatch import receiver
+from rest_framework.exceptions import ValidationError
+from datetime import datetime, timedelta, timezone
 from fyle_accounting_mappings.models import EmployeeMapping, Mapping, MappingSetting
-
-from apps.mappings.queues import (
-    async_auto_create_expense_field_mapping,
-    schedule_fyle_attributes_creation,
-)
-from apps.mappings.tasks import upload_attributes_to_fyle
-# TODO: Fix the naming convention when we remove the old schedule_or_delete_fyle_import_tasks import from helpers.py
-from apps.mappings.schedules import schedule_or_delete_fyle_import_tasks as new_schedule_or_delete_fyle_import_tasks
+from fyle_integrations_imports.models import ImportLog
+from apps.quickbooks_online.utils import QBOConnector
+from apps.workspaces.models import FyleCredential, QBOCredential, WorkspaceGeneralSettings
+from fyle.platform.exceptions import WrongParamsError
+from fyle_integrations_platform_connector import PlatformConnector
+from fyle_integrations_imports.modules.expense_custom_fields import ExpenseCustomField
 from apps.tasks.models import Error
 from apps.workspaces.apis.import_settings.triggers import ImportSettingsTrigger
-from apps.workspaces.models import WorkspaceGeneralSettings
 from apps.workspaces.utils import delete_cards_mapping_settings
+from apps.mappings.constants import SYNC_METHODS
+
+logger = logging.getLogger(__name__)
 
 
 @receiver(post_save, sender=Mapping)
@@ -50,12 +53,6 @@ def run_post_mapping_settings_triggers(sender, instance: MappingSetting, **kwarg
     """
     workspace_general_settings = WorkspaceGeneralSettings.objects.filter(workspace_id=instance.workspace_id).first()
 
-    if instance.source_field in ['PROJECT', 'COST_CENTER']:
-        new_schedule_or_delete_fyle_import_tasks(workspace_general_settings, instance)
-
-    if instance.is_custom:
-        schedule_fyle_attributes_creation(int(instance.workspace_id))
-
     if workspace_general_settings:
         delete_cards_mapping_settings(workspace_general_settings)
 
@@ -77,9 +74,74 @@ def run_pre_mapping_settings_triggers(sender, instance: MappingSetting, **kwargs
     instance.source_field = instance.source_field.upper().replace(' ', '_')
 
     if instance.source_field not in default_attributes:
-        upload_attributes_to_fyle(workspace_id=int(instance.workspace_id), qbo_attribute_type=instance.destination_field, fyle_attribute_type=instance.source_field, source_placeholder=instance.source_placeholder)
-
-        async_auto_create_expense_field_mapping(instance)
+        try:
+            workspace_id = int(instance.workspace_id)
+            # Checking is import_log exists or not if not create one
+            import_log, is_created = ImportLog.objects.get_or_create(
+                workspace_id=workspace_id,
+                attribute_type=instance.source_field,
+                defaults={
+                    'status': 'IN_PROGRESS'
+                }
+            )
+
+            last_successful_run_at = None
+            if import_log and not is_created:
+                last_successful_run_at = import_log.last_successful_run_at if import_log.last_successful_run_at else None
+                time_difference = datetime.now() - timedelta(minutes=32)
+                offset_aware_time_difference = time_difference.replace(tzinfo=timezone.utc)
+
+                # if the import_log is present and the last_successful_run_at is less than 30mins then we need to update it
+                # so that the schedule can run
+                if last_successful_run_at and offset_aware_time_difference\
+                    and (offset_aware_time_difference < last_successful_run_at):
+                    import_log.last_successful_run_at = offset_aware_time_difference
+                    last_successful_run_at = offset_aware_time_difference
+                    import_log.save()
+
+            qbo_credentials = QBOCredential.get_active_qbo_credentials(workspace_id)
+            qbo_connection = QBOConnector(credentials_object=qbo_credentials, workspace_id=workspace_id)
+
+            # Creating the expense_custom_field object with the correct last_successful_run_at value
+            expense_custom_field = ExpenseCustomField(
+                workspace_id=workspace_id,
+                source_field=instance.source_field,
+                destination_field=instance.destination_field,
+                sync_after=last_successful_run_at,
+                sdk_connection=qbo_connection,
+                destination_sync_methods=[SYNC_METHODS[instance.destination_field]]
+            )
+
+            fyle_credentials = FyleCredential.objects.get(workspace_id=workspace_id)
+            platform = PlatformConnector(fyle_credentials=fyle_credentials)
+
+            # setting the import_log status to IN_PROGRESS
+            import_log.status = 'IN_PROGRESS'
+            import_log.save()
+
+            expense_custom_field.construct_payload_and_import_to_fyle(platform, import_log)
+            expense_custom_field.sync_expense_attributes(platform)
+
+            # NOTE: We are not setting the import_log status to COMPLETE
+            # since the post_save trigger will run the import again in async manner
+
+        except WrongParamsError as error:
+            logger.error(
+                'Error while creating %s workspace_id - %s in Fyle %s %s',
+                instance.source_field, instance.workspace_id, error.message, {'error': error.response}
+            )
+            if error.response and 'message' in error.response:
+                raise ValidationError({
+                    'message': error.response['message'],
+                    'field_name': instance.source_field
+                })
+
+        # setting the import_log.last_successful_run_at to -30mins for the post_save_trigger
+        import_log = ImportLog.objects.filter(workspace_id=workspace_id, attribute_type=instance.source_field).first()
+        if import_log.last_successful_run_at:
+            last_successful_run_at = import_log.last_successful_run_at - timedelta(minutes=30)
+            import_log.last_successful_run_at = last_successful_run_at
+            import_log.save()
 
 
 @receiver(post_delete, sender=MappingSetting)

apps/mappings/tasks.py

@@ -1,12 +1,10 @@
 import logging
 from typing import Dict, List
 
-from dateutil import parser
 from django_q.tasks import Chain
 from fyle_accounting_mappings.models import DestinationAttribute, EmployeeMapping, ExpenseAttribute, Mapping, MappingSetting
 from fyle_integrations_platform_connector import PlatformConnector
 
-from apps.mappings.constants import FYLE_EXPENSE_SYSTEM_FIELDS
 from apps.mappings.exceptions import handle_import_exceptions
 from apps.mappings.models import GeneralMapping
 from apps.quickbooks_online.utils import QBOConnector
@@ -375,107 +373,6 @@ def create_fyle_tax_group_payload(qbo_attributes: List[DestinationAttribute], ex
     return fyle_tax_group_payload
 
 
-def create_fyle_expense_custom_field_payload(qbo_attributes: List[DestinationAttribute], workspace_id: int, fyle_attribute: str, platform: PlatformConnector, source_placeholder: str = None):
-    """
-    Create Fyle Expense Custom Field Payload from QBO Objects
-    :param workspace_id: Workspace ID
-    :param qbo_attributes: QBO Objects
-    :param fyle_attribute: Fyle Attribute
-    :return: Fyle Expense Custom Field Payload
-    """
-    fyle_expense_custom_field_options = []
-
-    if fyle_attribute.lower() not in FYLE_EXPENSE_SYSTEM_FIELDS:
-        existing_attribute = ExpenseAttribute.objects.filter(attribute_type=fyle_attribute, workspace_id=workspace_id).values_list('detail', flat=True).first()
-
-        custom_field_id = None
-        placeholder = None
-        is_mandatory = False
-        if existing_attribute is not None:
-            custom_field_id = existing_attribute['custom_field_id']
-            placeholder = existing_attribute['placeholder'] if 'placeholder' in existing_attribute else None
-            is_mandatory = existing_attribute['is_mandatory'] if 'is_mandatory' in existing_attribute else False
-            expense_field = platform.expense_custom_fields.get_by_id(custom_field_id)
-            fyle_expense_custom_field_options = expense_field['options']
-            last_imported_at = expense_field['updated_at']
-            qbo_attributes = [qbo_attribute for qbo_attribute in qbo_attributes if qbo_attribute.updated_at > parser.parse(last_imported_at)]
-
-        [fyle_expense_custom_field_options.append(qbo_attribute.value) for qbo_attribute in qbo_attributes]
-        fyle_expense_custom_field_options = list(set(fyle_expense_custom_field_options))
-        fyle_attribute = fyle_attribute.replace('_', ' ').title()
-
-        new_placeholder = None
-
-        # Here is the explanation of what's happening in the if-else ladder below
-        # source_field is the field that's save in mapping settings, this field user may or may not fill in the custom field form
-        # placeholder is the field that's saved in the detail column of destination attributes
-        # fyle_attribute is what we're constructing when both of these fields would not be available
-
-        if not (source_placeholder or placeholder):
-            # If source_placeholder and placeholder are both None, then we're creating adding a self constructed placeholder
-            new_placeholder = 'Select {0}'.format(fyle_attribute)
-        elif not source_placeholder and placeholder:
-            # If source_placeholder is None but placeholder is not, then we're choosing same place holder as 1 in detail section
-            new_placeholder = placeholder
-        elif source_placeholder and not placeholder:
-            # If source_placeholder is not None but placeholder is None, then we're choosing the placeholder as filled by user in form
-            new_placeholder = source_placeholder
-        else:
-            # Else, we're choosing the placeholder as filled by user in form or None
-            new_placeholder = source_placeholder
-
-        expense_custom_field_payload = {'field_name': fyle_attribute, 'type': 'SELECT', 'is_enabled': True, 'is_mandatory': is_mandatory, 'placeholder': new_placeholder, 'options': fyle_expense_custom_field_options, 'code': None}
-
-        if custom_field_id:
-            expense_custom_field_payload['id'] = custom_field_id
-
-        return expense_custom_field_payload
-
-
-def upload_attributes_to_fyle(workspace_id: int, qbo_attribute_type: str, fyle_attribute_type: str, source_placeholder: str = None):
-    """
-    Upload attributes to Fyle
-    """
-    fyle_credentials: FyleCredential = FyleCredential.objects.get(workspace_id=workspace_id)
-
-    platform = PlatformConnector(fyle_credentials)
-
-    qbo_attributes: List[DestinationAttribute] = DestinationAttribute.objects.filter(workspace_id=workspace_id, attribute_type=qbo_attribute_type)
-
-    if qbo_attributes.count():
-        qbo_attributes = remove_duplicates(qbo_attributes)
-
-        fyle_custom_field_payload = create_fyle_expense_custom_field_payload(qbo_attributes=qbo_attributes, workspace_id=workspace_id, fyle_attribute=fyle_attribute_type, platform=platform, source_placeholder=source_placeholder)
-
-        if fyle_custom_field_payload:
-            platform.expense_custom_fields.post(fyle_custom_field_payload)
-            platform.expense_custom_fields.sync()
-
-    return qbo_attributes
-
-
-@handle_import_exceptions(task_name='Auto Create Expense Fields Mappings')
-def auto_create_expense_fields_mappings(workspace_id: int, qbo_attribute_type: str, fyle_attribute_type: str, source_placeholder: str = None):
-    """
-    Create Fyle Attributes Mappings
-    :return: mappings
-    """
-    fyle_attributes = upload_attributes_to_fyle(workspace_id, qbo_attribute_type, fyle_attribute_type, source_placeholder)
-    if fyle_attributes:
-        Mapping.bulk_create_mappings(fyle_attributes, fyle_attribute_type, qbo_attribute_type, workspace_id)
-
-
-@handle_import_exceptions(task_name='Async Auto Create Custom Fields Mappings')
-def async_auto_create_custom_field_mappings(workspace_id):
-    mapping_settings = MappingSetting.objects.filter(is_custom=True, import_to_fyle=True, workspace_id=workspace_id).all()
-
-    if mapping_settings:
-        for mapping_setting in mapping_settings:
-            if mapping_setting.import_to_fyle:
-                sync_qbo_attribute(mapping_setting.destination_field, workspace_id)
-                auto_create_expense_fields_mappings(workspace_id, mapping_setting.destination_field, mapping_setting.source_field, mapping_setting.source_placeholder)
-
-
 def create_fyle_merchants_payload(vendors, existing_merchants_name):
     payload: List[str] = []
     for vendor in vendors: