Removing old code (#537)

* Removing old code

* adding test (#538)

* adding test

* lint fixes

* fix test

* sub module changes

* fixing test

* fixing test

* Migration script (#539)

apps/mappings/queues.py

@@ -1,7 +1,6 @@
-from datetime import datetime, timedelta
+from datetime import datetime
 
 from django_q.models import Schedule
-from django_q.tasks import async_task
 from fyle_accounting_mappings.models import MappingSetting
 
 from apps.mappings.models import GeneralMapping
@@ -12,23 +11,6 @@
 from apps.mappings.constants import SYNC_METHODS
 
 
-def async_auto_create_expense_field_mapping(mapping_setting: MappingSetting):
-    async_task('apps.mappings.tasks.auto_create_expense_fields_mappings', int(mapping_setting.workspace_id), mapping_setting.destination_field, mapping_setting.source_field)
-
-
-def schedule_fyle_attributes_creation(workspace_id: int):
-    mapping_settings = MappingSetting.objects.filter(is_custom=True, import_to_fyle=True, workspace_id=workspace_id).all()
-    if mapping_settings:
-        schedule, _ = Schedule.objects.get_or_create(
-            func='apps.mappings.tasks.async_auto_create_custom_field_mappings', args='{0}'.format(workspace_id), defaults={'schedule_type': Schedule.MINUTES, 'minutes': 24 * 60, 'next_run': datetime.now() + timedelta(hours=24)}
-        )
-    else:
-        schedule: Schedule = Schedule.objects.filter(func='apps.mappings.tasks.async_auto_create_custom_field_mappings', args='{0}'.format(workspace_id)).first()
-
-        if schedule:
-            schedule.delete()
-
-
 def schedule_bill_payment_creation(sync_fyle_to_qbo_payments, workspace_id):
     general_mappings: GeneralMapping = GeneralMapping.objects.filter(workspace_id=workspace_id).first()
     if general_mappings:

apps/mappings/tasks.py

@@ -1,12 +1,10 @@
 import logging
 from typing import Dict, List
 
-from dateutil import parser
 from django_q.tasks import Chain
 from fyle_accounting_mappings.models import DestinationAttribute, EmployeeMapping, ExpenseAttribute, Mapping, MappingSetting
 from fyle_integrations_platform_connector import PlatformConnector
 
-from apps.mappings.constants import FYLE_EXPENSE_SYSTEM_FIELDS
 from apps.mappings.exceptions import handle_import_exceptions
 from apps.mappings.models import GeneralMapping
 from apps.quickbooks_online.utils import QBOConnector
@@ -375,107 +373,6 @@ def create_fyle_tax_group_payload(qbo_attributes: List[DestinationAttribute], ex
     return fyle_tax_group_payload
 
 
-def create_fyle_expense_custom_field_payload(qbo_attributes: List[DestinationAttribute], workspace_id: int, fyle_attribute: str, platform: PlatformConnector, source_placeholder: str = None):
-    """
-    Create Fyle Expense Custom Field Payload from QBO Objects
-    :param workspace_id: Workspace ID
-    :param qbo_attributes: QBO Objects
-    :param fyle_attribute: Fyle Attribute
-    :return: Fyle Expense Custom Field Payload
-    """
-    fyle_expense_custom_field_options = []
-
-    if fyle_attribute.lower() not in FYLE_EXPENSE_SYSTEM_FIELDS:
-        existing_attribute = ExpenseAttribute.objects.filter(attribute_type=fyle_attribute, workspace_id=workspace_id).values_list('detail', flat=True).first()
-
-        custom_field_id = None
-        placeholder = None
-        is_mandatory = False
-        if existing_attribute is not None:
-            custom_field_id = existing_attribute['custom_field_id']
-            placeholder = existing_attribute['placeholder'] if 'placeholder' in existing_attribute else None
-            is_mandatory = existing_attribute['is_mandatory'] if 'is_mandatory' in existing_attribute else False
-            expense_field = platform.expense_custom_fields.get_by_id(custom_field_id)
-            fyle_expense_custom_field_options = expense_field['options']
-            last_imported_at = expense_field['updated_at']
-            qbo_attributes = [qbo_attribute for qbo_attribute in qbo_attributes if qbo_attribute.updated_at > parser.parse(last_imported_at)]
-
-        [fyle_expense_custom_field_options.append(qbo_attribute.value) for qbo_attribute in qbo_attributes]
-        fyle_expense_custom_field_options = list(set(fyle_expense_custom_field_options))
-        fyle_attribute = fyle_attribute.replace('_', ' ').title()
-
-        new_placeholder = None
-
-        # Here is the explanation of what's happening in the if-else ladder below
-        # source_field is the field that's save in mapping settings, this field user may or may not fill in the custom field form
-        # placeholder is the field that's saved in the detail column of destination attributes
-        # fyle_attribute is what we're constructing when both of these fields would not be available
-
-        if not (source_placeholder or placeholder):
-            # If source_placeholder and placeholder are both None, then we're creating adding a self constructed placeholder
-            new_placeholder = 'Select {0}'.format(fyle_attribute)
-        elif not source_placeholder and placeholder:
-            # If source_placeholder is None but placeholder is not, then we're choosing same place holder as 1 in detail section
-            new_placeholder = placeholder
-        elif source_placeholder and not placeholder:
-            # If source_placeholder is not None but placeholder is None, then we're choosing the placeholder as filled by user in form
-            new_placeholder = source_placeholder
-        else:
-            # Else, we're choosing the placeholder as filled by user in form or None
-            new_placeholder = source_placeholder
-
-        expense_custom_field_payload = {'field_name': fyle_attribute, 'type': 'SELECT', 'is_enabled': True, 'is_mandatory': is_mandatory, 'placeholder': new_placeholder, 'options': fyle_expense_custom_field_options, 'code': None}
-
-        if custom_field_id:
-            expense_custom_field_payload['id'] = custom_field_id
-
-        return expense_custom_field_payload
-
-
-def upload_attributes_to_fyle(workspace_id: int, qbo_attribute_type: str, fyle_attribute_type: str, source_placeholder: str = None):
-    """
-    Upload attributes to Fyle
-    """
-    fyle_credentials: FyleCredential = FyleCredential.objects.get(workspace_id=workspace_id)
-
-    platform = PlatformConnector(fyle_credentials)
-
-    qbo_attributes: List[DestinationAttribute] = DestinationAttribute.objects.filter(workspace_id=workspace_id, attribute_type=qbo_attribute_type)
-
-    if qbo_attributes.count():
-        qbo_attributes = remove_duplicates(qbo_attributes)
-
-        fyle_custom_field_payload = create_fyle_expense_custom_field_payload(qbo_attributes=qbo_attributes, workspace_id=workspace_id, fyle_attribute=fyle_attribute_type, platform=platform, source_placeholder=source_placeholder)
-
-        if fyle_custom_field_payload:
-            platform.expense_custom_fields.post(fyle_custom_field_payload)
-            platform.expense_custom_fields.sync()
-
-    return qbo_attributes
-
-
-@handle_import_exceptions(task_name='Auto Create Expense Fields Mappings')
-def auto_create_expense_fields_mappings(workspace_id: int, qbo_attribute_type: str, fyle_attribute_type: str, source_placeholder: str = None):
-    """
-    Create Fyle Attributes Mappings
-    :return: mappings
-    """
-    fyle_attributes = upload_attributes_to_fyle(workspace_id, qbo_attribute_type, fyle_attribute_type, source_placeholder)
-    if fyle_attributes:
-        Mapping.bulk_create_mappings(fyle_attributes, fyle_attribute_type, qbo_attribute_type, workspace_id)
-
-
-@handle_import_exceptions(task_name='Async Auto Create Custom Fields Mappings')
-def async_auto_create_custom_field_mappings(workspace_id):
-    mapping_settings = MappingSetting.objects.filter(is_custom=True, import_to_fyle=True, workspace_id=workspace_id).all()
-
-    if mapping_settings:
-        for mapping_setting in mapping_settings:
-            if mapping_setting.import_to_fyle:
-                sync_qbo_attribute(mapping_setting.destination_field, workspace_id)
-                auto_create_expense_fields_mappings(workspace_id, mapping_setting.destination_field, mapping_setting.source_field, mapping_setting.source_placeholder)
-
-
 def create_fyle_merchants_payload(vendors, existing_merchants_name):
     payload: List[str] = []
     for vendor in vendors:

apps/quickbooks_online/actions.py

@@ -72,7 +72,7 @@ def refresh_quickbooks_dimensions(workspace_id: int):
     chain = Chain()
 
     for mapping_setting in mapping_settings:
-        if mapping_setting.source_field in ['PROJECT', 'COST_CENTER']:
+        if mapping_setting.source_field in ['PROJECT', 'COST_CENTER'] or mapping_setting.is_custom:
             chain.append(
                 'fyle_integrations_imports.tasks.trigger_import_via_schedule',
                 workspace_id,
@@ -84,10 +84,8 @@ def refresh_quickbooks_dimensions(workspace_id: int):
                 get_auto_sync_permission(workspace_general_settings, mapping_setting),
                 False,
                 None,
-                False
+                mapping_setting.is_custom
             )
-        elif mapping_setting.is_custom:
-            chain.append('apps.mappings.tasks.async_auto_create_custom_field_mappings', int(workspace_id))
 
     if chain.length() > 0:
         chain.run()

scripts/python/create-update-new-custom-fields-import.py

@@ -0,0 +1,51 @@
+from django.db import transaction
+import random
+from datetime import datetime, timedelta
+from django_q.models import Schedule
+from fyle_accounting_mappings.models import MappingSetting
+
+existing_import_enabled_schedules = Schedule.objects.filter(
+    func__in=['apps.mappings.tasks.async_auto_create_custom_field_mappings']
+).values('args')
+
+try:
+    with transaction.atomic():
+        count = 0
+        for schedule in existing_import_enabled_schedules:
+            random_number = random.randint(1, 23)
+            mapping_setting = MappingSetting.objects.filter(workspace_id=schedule['args'], import_to_fyle=True, is_custom=True).first()
+            if mapping_setting:
+                print('Creating schedule for workspace_id: ', schedule['args'])
+                # adding the new schedule
+                Schedule.objects.update_or_create(
+                    func='apps.mappings.queues.construct_tasks_and_chain_import_fields_to_fyle',
+                    args=schedule['args'],
+                    defaults={
+                        'schedule_type': Schedule.MINUTES,
+                        'minutes':24 * 60,
+                        'next_run':datetime.now() + timedelta(hours=random_number)
+                    }
+                )
+                # deleting the old schedule
+                Schedule.objects.filter(
+                    func='apps.mappings.tasks.async_auto_create_custom_field_mappings',
+                    args=schedule['args']
+                ).delete()
+                count += 1
+        print("""
+
+            Schedules created
+
+        """)
+        print(count)
+        # remove this sanity check after running this script
+        raise Exception("This is a sanity check")
+except Exception as e:
+    print(e)
+
+
+# Run this in sql
+# select * from django_q_schedule where func = 'apps.mappings.tasks.async_auto_create_custom_field_mappings';
+# --rows should be 0
+# If not check the workspace_id and delete the row
+# delete from django_q_schedule where func = 'apps.mappings.tasks.async_auto_create_custom_field_mappings' and args = 'workspace_id';