Refactor deps import (#180)

* Delete purchase_invoice, line_items on failed exports from hh2, type-check for current_state * Refactor deps schedule to run post Job import * Remove dep setting trigger, add logger * Remove mock object * Add details while logging * Fyle Card <> Vendor Mapping setup (#181) * Fyle Card <> Vendor Mapping setup * Added script to add mapping_settings * Fix post release script * Modified script, added additional test case * lint fix * modify post-release script * Disable sage fields (#183) * Fyle Card <> Vendor Mapping setup * Added script to add mapping_settings * Fix post release script * Projects and Deps fields disable v1 * Modified script, added additional test case * lint fix * modify post-release script * bump accounting-mapping version * modify the variable_name, add conditional update * Add example objects * Added loggers * Added test cases * Modify test case, add filter in dep field settings * Dependent Field optimizations (#185) * Fyle Card <> Vendor Mapping setup * Added script to add mapping_settings * Fix post release script * Projects and Deps fields disable v1 * Modified script, added additional test case * lint fix * modify post-release script * bump accounting-mapping version * modify the variable_name, add conditional update * Add example objects * Added loggers * Added test cases * Dependent Field optimizations * fix failing test * Fix post-release script * Fix the merged issues * Add filters and change log
fylein · Jun 11, 2024 · 6d2ceaa · 6d2ceaa
1 parent 3749326
commit 6d2ceaa
Show file tree

Hide file tree

Showing 20 changed files with 507 additions and 96 deletions.
diff --git a/apps/accounting_exports/models.py b/apps/accounting_exports/models.py
@@ -47,7 +47,7 @@ def _group_expenses(expenses: List[Expense], export_setting: ExportSetting, fund
     reimbursable_expense_date = export_setting.reimbursable_expense_date
 
     default_fields = ['employee_email', 'fund_source']
-    report_grouping_fields = ['report_id', 'claim_number']
+    report_grouping_fields = ['report_id', 'claim_number', 'corporate_card_id']
     expense_grouping_fields = ['expense_id', 'expense_number']
 
     # Define a mapping for fund sources and their associated group fields

diff --git a/apps/fyle/signals.py b/apps/fyle/signals.py
@@ -3,15 +3,13 @@
 """
 import logging
 
-from django.db.models.signals import post_save, pre_save
+from django.db.models.signals import pre_save
 from django.dispatch import receiver
 
 from fyle_integrations_platform_connector import PlatformConnector
 from apps.workspaces.models import FyleCredential
 from apps.fyle.models import DependentFieldSetting
 from apps.sage300.dependent_fields import create_dependent_custom_field_in_fyle
-from apps.mappings.imports.schedules import schedule_or_delete_dependent_field_tasks
-
 
 logger = logging.getLogger(__name__)
 logger.level = logging.INFO
@@ -51,13 +49,3 @@ def run_pre_save_dependent_field_settings_triggers(sender, instance: DependentFi
         parent_field_id=instance.cost_code_field_id,
     )
     instance.cost_category_field_id = cost_category['data']['id']
-
-
-@receiver(post_save, sender=DependentFieldSetting)
-def run_post_save_dependent_field_settings_triggers(sender, instance: DependentFieldSetting, **kwargs):
-    """
-    :param sender: Sender Class
-    :param instance: Row instance of Sender Class
-    :return: None
-    """
-    schedule_or_delete_dependent_field_tasks(instance.workspace_id)
diff --git a/apps/mappings/imports/queues.py b/apps/mappings/imports/queues.py
@@ -1,6 +1,7 @@
 from django_q.tasks import Chain
 from fyle_accounting_mappings.models import MappingSetting
 from apps.workspaces.models import ImportSetting
+from apps.fyle.models import DependentFieldSetting
 
 
 def chain_import_fields_to_fyle(workspace_id):
@@ -11,6 +12,9 @@ def chain_import_fields_to_fyle(workspace_id):
     mapping_settings = MappingSetting.objects.filter(workspace_id=workspace_id, import_to_fyle=True)
     custom_field_mapping_settings = MappingSetting.objects.filter(workspace_id=workspace_id, is_custom=True, import_to_fyle=True)
     import_settings = ImportSetting.objects.get(workspace_id=workspace_id)
+    dependent_field_settings = DependentFieldSetting.objects.filter(workspace_id=workspace_id, is_import_enabled=True).first()
+    project_mapping = MappingSetting.objects.filter(workspace_id=workspace_id, source_field='PROJECT', import_to_fyle=True).first()
+
     chain = Chain()
 
     if import_settings.import_categories:
@@ -47,5 +51,11 @@ def chain_import_fields_to_fyle(workspace_id):
             True
         )
 
+    if project_mapping and dependent_field_settings:
+        chain.append(
+            'apps.mappings.imports.tasks.auto_import_and_map_fyle_fields',
+            workspace_id
+        )
+
     if chain.length() > 0:
         chain.run()
diff --git a/apps/mappings/imports/schedules.py b/apps/mappings/imports/schedules.py
@@ -1,41 +1,9 @@
 from datetime import datetime
 from django_q.models import Schedule
 from fyle_accounting_mappings.models import MappingSetting
-
-from apps.fyle.models import DependentFieldSetting
 from apps.workspaces.models import ImportSetting
 
 
-def schedule_or_delete_dependent_field_tasks(workspace_id: int):
-    """
-    :param configuration: Workspace Configuration Instance
-    :return: None
-    """
-    project_mapping = MappingSetting.objects.filter(
-        source_field='PROJECT',
-        workspace_id=workspace_id,
-        import_to_fyle=True
-    ).first()
-    dependent_fields = DependentFieldSetting.objects.filter(workspace_id=workspace_id, is_import_enabled=True).first()
-
-    if project_mapping and dependent_fields:
-        start_datetime = datetime.now()
-        Schedule.objects.update_or_create(
-            func='apps.mappings.imports.tasks.auto_import_and_map_fyle_fields',
-            args='{}'.format(workspace_id),
-            defaults={
-                'schedule_type': Schedule.MINUTES,
-                'minutes': 24 * 60,
-                'next_run': start_datetime
-            }
-        )
-    elif not (project_mapping and dependent_fields):
-        Schedule.objects.filter(
-            func='apps.mappings.imports.tasks.auto_import_and_map_fyle_fields',
-            args='{}'.format(workspace_id)
-        ).delete()
-
-
 def schedule_or_delete_fyle_import_tasks(import_settings: ImportSetting, mapping_setting_instance: MappingSetting = None):
     """
     Schedule or delete Fyle import tasks based on the import settingss.
@@ -78,6 +46,3 @@ def schedule_or_delete_fyle_import_tasks(import_settings: ImportSetting, mapping
             func='apps.mappings.imports.queues.chain_import_fields_to_fyle',
             args='{}'.format(import_settings.workspace_id)
         ).delete()
-
-    # Schedule or delete dependent field tasks
-    schedule_or_delete_dependent_field_tasks(import_settings.workspace_id)
diff --git a/apps/mappings/imports/tasks.py b/apps/mappings/imports/tasks.py
@@ -1,14 +1,15 @@
+import logging
 from django_q.tasks import Chain
 
-from fyle_accounting_mappings.models import MappingSetting
-
 from apps.mappings.models import ImportLog
 from apps.mappings.imports.modules.categories import Category
 from apps.mappings.imports.modules.projects import Project
 from apps.mappings.imports.modules.cost_centers import CostCenter
 from apps.mappings.imports.modules.merchants import Merchant
 from apps.mappings.imports.modules.expense_custom_fields import ExpenseCustomField
-from apps.fyle.models import DependentFieldSetting
+
+logger = logging.getLogger(__name__)
+logger.level = logging.INFO
 
 SOURCE_FIELD_CLASS_MAP = {
     'CATEGORY': Category,
@@ -41,20 +42,20 @@ def auto_import_and_map_fyle_fields(workspace_id):
     """
     Auto import and map fyle fields
     """
-    project_mapping = MappingSetting.objects.filter(
-        source_field='PROJECT',
+    import_log = ImportLog.objects.filter(
         workspace_id=workspace_id,
-        import_to_fyle=True
+        attribute_type = 'PROJECT'
     ).first()
-    dependent_fields = DependentFieldSetting.objects.filter(workspace_id=workspace_id, is_import_enabled=True).first()
 
     chain = Chain()
 
-    if project_mapping and dependent_fields:
-        chain.append('apps.mappings.tasks.sync_sage300_attributes', 'JOB', workspace_id)
-        chain.append('apps.mappings.tasks.sync_sage300_attributes', 'COST_CODE', workspace_id)
-        chain.append('apps.mappings.tasks.sync_sage300_attributes', 'COST_CATEGORY', workspace_id)
-        chain.append('apps.sage300.dependent_fields.import_dependent_fields_to_fyle', workspace_id)
+    chain.append('apps.mappings.tasks.sync_sage300_attributes', 'JOB', workspace_id)
+    chain.append('apps.mappings.tasks.sync_sage300_attributes', 'COST_CODE', workspace_id)
+    chain.append('apps.mappings.tasks.sync_sage300_attributes', 'COST_CATEGORY', workspace_id)
+    chain.append('apps.sage300.dependent_fields.import_dependent_fields_to_fyle', workspace_id)
+
+    if import_log and import_log.status != 'COMPLETE':
+        logger.error(f"Project Import is in {import_log.status} state in WORKSPACE_ID: {workspace_id} with error {str(import_log.error_log)}")
 
     if chain.length() > 0:
         chain.run()
diff --git a/apps/sage300/dependent_fields.py b/apps/sage300/dependent_fields.py
@@ -68,8 +68,7 @@ def create_dependent_custom_field_in_fyle(workspace_id: int, fyle_attribute_type
     return platform.expense_custom_fields.post(expense_custom_field_payload)
 
 
-def post_dependent_cost_code(dependent_field_setting: DependentFieldSetting, platform: PlatformConnector, filters: Dict) -> List[str]:
-
+def post_dependent_cost_code(dependent_field_setting: DependentFieldSetting, platform: PlatformConnector, filters: Dict, is_enabled: bool = True) -> List[str]:
     projects = CostCategory.objects.filter(**filters).values('job_name').annotate(cost_codes=ArrayAgg('cost_code_name', distinct=True))
     projects_from_categories = [project['job_name'] for project in projects]
     posted_cost_codes = []
@@ -91,20 +90,24 @@ def post_dependent_cost_code(dependent_field_setting: DependentFieldSetting, pla
                     'parent_expense_field_value': project['job_name'],
                     'expense_field_id': dependent_field_setting.cost_code_field_id,
                     'expense_field_value': cost_code,
-                    'is_enabled': True
+                    'is_enabled': is_enabled
                 })
                 cost_code_names.append(cost_code)
 
         if payload:
             sleep(0.2)
-            platform.dependent_fields.bulk_post_dependent_expense_field_values(payload)
-            posted_cost_codes.extend(cost_code_names)
+            try:
+                platform.dependent_fields.bulk_post_dependent_expense_field_values(payload)
+                posted_cost_codes.extend(cost_code_names)
+            except Exception as exception:
+                logger.error(f'Exception while posting dependent cost code | Error: {exception} | Payload: {payload}')
+                raise
 
     return posted_cost_codes
 
 
 def post_dependent_cost_type(dependent_field_setting: DependentFieldSetting, platform: PlatformConnector, filters: Dict):
-    cost_categories = CostCategory.objects.filter(**filters).values('cost_code_name').annotate(cost_categories=ArrayAgg('name', distinct=True))
+    cost_categories = CostCategory.objects.filter(is_imported=False, **filters).values('cost_code_name').annotate(cost_categories=ArrayAgg('name', distinct=True))
 
     for category in cost_categories:
         payload = [
@@ -119,7 +122,12 @@ def post_dependent_cost_type(dependent_field_setting: DependentFieldSetting, pla
 
         if payload:
             sleep(0.2)
-            platform.dependent_fields.bulk_post_dependent_expense_field_values(payload)
+            try:
+                platform.dependent_fields.bulk_post_dependent_expense_field_values(payload)
+                CostCategory.objects.filter(cost_code_name=category['cost_code_name']).update(is_imported=True)
+            except Exception as exception:
+                logger.error(f'Exception while posting dependent cost type | Error: {exception} | Payload: {payload}')
+                raise
 
 
 def post_dependent_expense_field_values(workspace_id: int, dependent_field_setting: DependentFieldSetting, platform: PlatformConnector = None):

diff --git a/apps/sage300/exports/base_model.py b/apps/sage300/exports/base_model.py
@@ -3,7 +3,7 @@
 
 from django.db import models
 from django.db.models import Sum
-from fyle_accounting_mappings.models import ExpenseAttribute, Mapping, MappingSetting, EmployeeMapping, DestinationAttribute
+from fyle_accounting_mappings.models import ExpenseAttribute, Mapping, MappingSetting, EmployeeMapping
 
 from apps.accounting_exports.models import AccountingExport
 from apps.fyle.models import DependentFieldSetting, Expense
@@ -60,7 +60,6 @@ def get_expense_purpose(workspace_id, lineitem: Expense, category: str, advance_
     def get_vendor_id(accounting_export: AccountingExport):
         # Retrieve export settings for the given workspace
         export_settings = ExportSetting.objects.get(workspace_id=accounting_export.workspace_id)
-
         # Extract the description from the accounting export
         description = accounting_export.description
 
@@ -81,22 +80,22 @@ def get_vendor_id(accounting_export: AccountingExport):
         # Check if the fund source is 'CCC'
         elif accounting_export.fund_source == 'CCC':
             # Retrieve the vendor from the first expense
-            expense_vendor = accounting_export.expenses.first().vendor
+            vendor_id = None
+            corporate_card_id = accounting_export.expenses.first().corporate_card_id
 
-            # Query DestinationAttribute for the vendor with case-insensitive search
-            if expense_vendor:
-                vendor = DestinationAttribute.objects.filter(
+            if corporate_card_id:
+                vendor_mapping = Mapping.objects.filter(
                     workspace_id=accounting_export.workspace_id,
-                    value__icontains=expense_vendor,
-                    attribute_type='VENDOR'
-                ).values_list('destination_id', flat=True).first()
-                if not vendor:
-                    vendor = export_settings.default_vendor_id
-            else:
-                vendor = export_settings.default_vendor_id
+                    source_type='CORPORATE_CARD',
+                    destination_type='VENDOR',
+                    source__source_id=corporate_card_id
+                ).first()
 
-            # Update vendor_id with the retrieved vendor or default to export settings
-            vendor_id = vendor
+                if vendor_mapping:
+                    vendor_id = vendor_mapping.destination.destination_id
+
+            if not vendor_id:
+                vendor_id = export_settings.default_vendor_id
 
         # Return the determined vendor_id
         return vendor_id

diff --git a/apps/sage300/helpers.py b/apps/sage300/helpers.py
@@ -2,11 +2,16 @@
 from datetime import datetime, timezone
 import logging
 
+from typing import Dict
 from django.utils.module_loading import import_string
 
-from apps.workspaces.models import Workspace, Sage300Credential
+from apps.workspaces.models import Workspace, Sage300Credential, FyleCredential
 from apps.mappings.models import Version
-
+from fyle_accounting_mappings.models import ExpenseAttribute
+from fyle_integrations_platform_connector import PlatformConnector
+from apps.sage300.models import CostCategory
+from apps.fyle.models import DependentFieldSetting
+from apps.sage300.dependent_fields import post_dependent_cost_code
 
 logger = logging.getLogger(__name__)
 logger.level = logging.INFO
@@ -62,3 +67,96 @@ def sync_dimensions(sage300_credential: Sage300Credential, workspace_id: int) ->
         except Exception as exception:
             # Log any exceptions that occur during synchronization
             logger.info(exception)
+
+
+def disable_projects(workspace_id: int, projects_to_disable: Dict):
+    """
+    Disable projects in Fyle when the projects are updated in Sage 300.
+    This is a callback function that is triggered from accounting_mappings.
+    projects_to_disable object format:
+    {
+        'destination_id': {
+            'value': 'old_project_name',
+            'updated_value': 'new_project_name'
+        }
+    }
+
+    """
+    fyle_credentials = FyleCredential.objects.get(workspace_id=workspace_id)
+    platform = PlatformConnector(fyle_credentials=fyle_credentials)
+
+    filters = {
+        'workspace_id': workspace_id,
+        'attribute_type': 'PROJECT',
+        'value__in': [projects_map['value'] for projects_map in projects_to_disable.values()]
+    }
+
+    # Expense attribute value map is as follows: {old_project_name: destination_id}
+    expense_attribute_value_map = {v['value']: k for k, v in projects_to_disable.items()}
+
+    expense_attributes = ExpenseAttribute.objects.filter(**filters)
+
+    bulk_payload = []
+    for expense_attribute in expense_attributes:
+        code = expense_attribute_value_map.get(expense_attribute.value, None)
+        if code:
+            payload = {
+                'name': expense_attribute.value,
+                'code': code,
+                'description': 'Sage 300 Project - {0}, Id - {1}'.format(
+                    expense_attribute.value,
+                    code
+                ),
+                'is_enabled': False,
+                'id': expense_attribute.source_id
+            }
+        else:
+            logger.error(f"Project with value {expense_attribute.value} not found | WORKSPACE_ID: {workspace_id}")
+
+        bulk_payload.append(payload)
+
+    sync_after = datetime.now(timezone.utc)
+
+    if bulk_payload:
+        logger.info(f"Disabling Projects in Fyle | WORKSPACE_ID: {workspace_id} | COUNT: {len(bulk_payload)}")
+        platform.projects.post_bulk(bulk_payload)
+        platform.projects.sync(sync_after=sync_after)
+    else:
+        logger.info(f"No Projects to Disable in Fyle | WORKSPACE_ID: {workspace_id}")
+
+    update_and_disable_cost_code(workspace_id, projects_to_disable, platform)
+
+
+def update_and_disable_cost_code(workspace_id: int, cost_codes_to_disable: Dict, platform: PlatformConnector):
+    """
+    Update the job_name in CostCategory and disable the old cost code in Fyle
+    """
+    dependent_field_setting = DependentFieldSetting.objects.filter(is_import_enabled=True, workspace_id=workspace_id).first()
+
+    if dependent_field_setting:
+        filters = {
+            'job_id__in':list(cost_codes_to_disable.keys()),
+            'workspace_id': workspace_id
+        }
+
+        # This call will disable the cost codes in Fyle that has old project name
+        posted_cost_codes = post_dependent_cost_code(dependent_field_setting, platform, filters, is_enabled=False)
+
+        logger.info(f"Disabled Cost Codes in Fyle | WORKSPACE_ID: {workspace_id} | COUNT: {len(posted_cost_codes)}")
+
+        # here we are updating the CostCategory with the new project name
+        bulk_update_payload = []
+        for destination_id, value in cost_codes_to_disable.items():
+            cost_categories = CostCategory.objects.filter(
+                workspace_id=workspace_id,
+                job_id=destination_id
+            ).exclude(job_name=value['updated_value'])
+
+            for cost_category in cost_categories:
+                cost_category.job_name = value['updated_value']
+                cost_category.updated_at = datetime.now(timezone.utc)
+                bulk_update_payload.append(cost_category)
+
+        if bulk_update_payload:
+            logger.info(f"Updating Cost Categories | WORKSPACE_ID: {workspace_id} | COUNT: {len(bulk_update_payload)}")
+            CostCategory.objects.bulk_update(bulk_update_payload, ['job_name', 'updated_at'], batch_size=50)