Setting Employee Mapping error, resolving and export exception decorator

apps/mappings/signals.py

@@ -5,18 +5,46 @@
 from django.dispatch import receiver
 from django_q.tasks import async_task
 
-from fyle_accounting_mappings.models import MappingSetting
+from fyle_accounting_mappings.models import MappingSetting, EmployeeMapping, Mapping, CategoryMapping
 
 from apps.mappings.tasks import upload_attributes_to_fyle, schedule_cost_centers_creation,\
     schedule_fyle_attributes_creation
 from apps.mappings.helpers import schedule_or_delete_fyle_import_tasks
 from apps.netsuite.helpers import schedule_payment_sync
 from apps.workspaces.models import Configuration
 from apps.workspaces.tasks import delete_cards_mapping_settings
+from apps.tasks.models import Error
 
 from .models import GeneralMapping, SubsidiaryMapping
 from .tasks import schedule_auto_map_ccc_employees
 
+@receiver(post_save, sender=Mapping)
+def resolve_post_mapping_errors(sender, instance: Mapping, **kwargs):
+    """
+    Resolve errors after mapping is created
+    """
+    if instance.source_type == 'TAX_GROUP':
+        Error.objects.filter(expense_attribute_id=instance.source_id).update(
+            is_resolved=True
+        )
+
+@receiver(post_save, sender=CategoryMapping)
+def resolve_post_category_mapping_errors(sender, instance: Mapping, **kwargs):
+    """
+    Resolve errors after mapping is created
+    """
+    Error.objects.filter(expense_attribute_id=instance.source_category_id).update(
+        is_resolved=True
+    )
+
+@receiver(post_save, sender=EmployeeMapping)
+def resolve_post_employees_mapping_errors(sender, instance: Mapping, **kwargs):
+    """
+    Resolve errors after mapping is created 
+    """
+    Error.objects.filter(expense_attribute_id=instance.source_employee_id).update(
+        is_resolved=True
+    )
 
 @receiver(post_save, sender=SubsidiaryMapping)
 def run_post_subsidiary_mappings(sender, instance: SubsidiaryMapping, **kwargs):

apps/mappings/tasks.py

@@ -13,20 +13,78 @@
 from fyle.platform.exceptions import WrongParamsError, InvalidTokenError
 
 from fyle_accounting_mappings.models import Mapping, MappingSetting, ExpenseAttribute, DestinationAttribute,\
-    CategoryMapping
+    CategoryMapping, EmployeeMapping
 from fyle_accounting_mappings.helpers import EmployeesAutoMappingHelper
 
 from fyle_integrations_platform_connector import PlatformConnector
 from apps.mappings.models import GeneralMapping
 from apps.netsuite.connector import NetSuiteConnector
 from apps.workspaces.models import NetSuiteCredentials, FyleCredential, Configuration, Workspace
+from apps.tasks.models import Error
 
 from .exceptions import handle_exceptions
 from .constants import FYLE_EXPENSE_SYSTEM_FIELDS, DEFAULT_NETSUITE_IMPORT_TYPES
 
 logger = logging.getLogger(__name__)
 logger.level = logging.INFO
 
+def get_mapped_attributes_ids(source_attribute_type: str, destination_attribute_type: str, errored_attribute_ids: List[int]):
+
+    mapped_attribute_ids = []
+
+    if source_attribute_type == "TAX_GROUP":
+        mapped_attribute_ids: List[int] = Mapping.objects.filter(
+            source_id__in=errored_attribute_ids
+        ).values_list('source_id', flat=True)
+
+    elif source_attribute_type == "EMPLOYEE":
+        params = {
+            'source_employee_id__in': errored_attribute_ids,
+        }
+
+        if destination_attribute_type == "EMPLOYEE":
+            params['destination_employee_id__isnull'] = False
+        else:
+            params['destination_vendor_id__isnull'] = False
+        mapped_attribute_ids: List[int] = EmployeeMapping.objects.filter(
+            **params
+        ).values_list('source_employee_id', flat=True)
+
+    elif source_attribute_type == "CATEGORY":
+        params = {
+            'source_category_id__in': errored_attribute_ids,
+        }
+
+        if destination_attribute_type == 'EXPENSE_TYPE':
+            params['destination_expense_head_id__isnull'] = False
+        else:
+            params['destination_account_id__isnull'] =  False
+
+        mapped_attribute_ids: List[int] = CategoryMapping.objects.filter(
+            **params
+        ).values_list('source_category_id', flat=True)
+
+    return mapped_attribute_ids
+
+
+def resolve_expense_attribute_errors(
+    source_attribute_type: str, workspace_id: int, destination_attribute_type: str = None):
+    """
+    Resolve Expense Attribute Errors
+    :return: None
+    """
+    errored_attribute_ids: List[int] = Error.objects.filter(
+        is_resolved=False,
+        workspace_id=workspace_id,
+        type='{}_MAPPING'.format(source_attribute_type)
+    ).values_list('expense_attribute_id', flat=True)
+
+    if errored_attribute_ids:
+        mapped_attribute_ids = get_mapped_attributes_ids(source_attribute_type, destination_attribute_type, errored_attribute_ids)
+
+        if mapped_attribute_ids:
+            Error.objects.filter(expense_attribute_id__in=mapped_attribute_ids).update(is_resolved=True)
+
 def remove_duplicates(ns_attributes: List[DestinationAttribute]):
     unique_attributes = []
 
@@ -467,6 +525,9 @@ def post_tax_groups(platform_connection: PlatformConnector, workspace_id: int):
 
     platform_connection.tax_groups.sync()
     Mapping.bulk_create_mappings(netsuite_attributes, 'TAX_GROUP', 'TAX_ITEM', workspace_id)
+    resolve_expense_attribute_errors(
+        source_attribute_type='TAX_GROUP', workspace_id=workspace_id
+    )
 
 @handle_exceptions(task_name='Import Category to Fyle and Auto Create Mappings')
 def auto_create_category_mappings(workspace_id):
@@ -505,6 +566,12 @@ def auto_create_category_mappings(workspace_id):
     if reimbursable_expenses_object == 'EXPENSE REPORT' and \
         corporate_credit_card_expenses_object in ('BILL', 'JOURNAL ENTRY', 'CREDIT CARD CHARGE'):
         bulk_create_ccc_category_mappings(workspace_id)
+
+    resolve_expense_attribute_errors(
+            source_attribute_type="CATEGORY", 
+            destination_attribute_type=reimbursable_destination_type, 
+            workspace_id=workspace_id
+    )
 
 
 def auto_import_and_map_fyle_fields(workspace_id):
@@ -657,6 +724,11 @@ def async_auto_map_employees(workspace_id: int):
         netsuite_connection.sync_vendors()
 
     EmployeesAutoMappingHelper(workspace_id, destination_type, employee_mapping_preference).reimburse_mapping()
+    resolve_expense_attribute_errors(
+        source_attribute_type="EMPLOYEE",
+        workspace_id=workspace_id,
+        destination_attribute_type=destination_type,
+        )
 
 
 def schedule_auto_map_employees(employee_mapping_preference: str, workspace_id: int):

apps/netsuite/actions.py

@@ -0,0 +1,25 @@
+from apps.tasks.models import TaskLog
+from apps.workspaces.models import LastExportDetail
+from django.db.models import Q
+
+
+def update_last_export_details(workspace_id):
+    last_export_detail = LastExportDetail.objects.get(workspace_id=workspace_id)
+
+    failed_exports = TaskLog.objects.filter(
+        ~Q(type__in=['CREATING_VENDOR_PAYMENT','FETCHING_EXPENSES']), workspace_id=workspace_id, status__in=['FAILED', 'FATAL']
+    ).count()
+
+    successful_exports = TaskLog.objects.filter(
+        ~Q(type__in=['CREATING_VENDOR_PAYMENT', 'FETCHING_EXPENSES']),
+        workspace_id=workspace_id,
+        status='COMPLETE',
+        updated_at__gt=last_export_detail.last_exported_at
+    ).count()
+
+    last_export_detail.failed_expense_groups_count = failed_exports
+    last_export_detail.successful_expense_groups_count = successful_exports
+    last_export_detail.total_expense_groups_count = failed_exports + successful_exports
+    last_export_detail.save()
+
+    return last_export_detail

apps/netsuite/exceptions.py

@@ -0,0 +1,160 @@
+import logging
+import json
+import traceback
+
+from apps.fyle.models import ExpenseGroup
+from apps.tasks.models import TaskLog, Error
+from apps.workspaces.models import LastExportDetail, NetSuiteCredentials
+
+from netsuitesdk.internal.exceptions import NetSuiteRequestError
+from netsuitesdk import NetSuiteRateLimitError, NetSuiteLoginError
+from fyle_netsuite_api.exceptions import BulkError
+
+from .actions import update_last_export_details
+
+from django.db.models import Q
+
+logger = logging.getLogger(__name__)
+logger.level = logging.INFO
+
+netsuite_error_message = 'NetSuite System Error'
+
+def __handle_netsuite_connection_error(expense_group: ExpenseGroup, task_log: TaskLog) -> None:
+    logger.info(
+        'NetSuite Credentials not found for workspace_id %s / expense group %s',
+        expense_group.id,
+        expense_group.workspace_id
+    )
+    detail = {
+        'message': 'NetSuite Account not connected'
+    }
+
+    Error.objects.update_or_create(
+        workspace_id=expense_group.workspace_id,
+        expense_group=expense_group,
+        defaults={
+            'type': 'NETSUITE_ERROR',
+            'error_title': netsuite_error_message,
+            'error_detail': detail['message'],
+            'is_resolved': False
+        })
+
+    task_log.status = 'FAILED'
+    task_log.detail = detail
+
+    task_log.save()
+
+
+def __log_error(task_log: TaskLog) -> None:
+    logger.exception('Something unexpected happened workspace_id: %s %s', task_log.workspace_id, task_log.detail)
+
+
+def handle_netsuite_exceptions(payment=False):
+    def decorator(func):
+        def wrapper(*args):
+            if payment:
+                entity_object = args[0]
+                workspace_id = args[1]
+                object_type = args[2]
+                task_log, _ = TaskLog.objects.update_or_create(
+                workspace_id=workspace_id,
+                task_id='PAYMENT_{}'.format(entity_object['unique_id']),
+                defaults={
+                    'status': 'IN_PROGRESS',
+                    'type': 'CREATING_VENDOR_PAYMENT'
+                    }
+                )
+            else:
+                expense_group = args[0]
+                task_log_id = args[1]
+                task_log = TaskLog.objects.get(id=task_log_id)
+                last_export = args[2]
+
+            try:
+                func(*args)
+
+            except NetSuiteCredentials.DoesNotExist:
+                if payment:
+                    logger.info(
+                        'NetSuite Credentials not found for workspace_id %s',
+                        workspace_id
+                    )
+                    detail = {
+                        'message': 'NetSuite Account not connected'
+                    }
+                    task_log.status = 'FAILED'
+                    task_log.detail = detail
+
+                    task_log.save()
+                else:
+                    __handle_netsuite_connection_error(expense_group, task_log)
+
+            except (NetSuiteRequestError, NetSuiteLoginError) as exception:
+                all_details = []
+                logger.info({'error': exception})
+                detail = json.dumps(exception.__dict__)
+                detail = json.loads(detail)
+                task_log.status = 'FAILED'
+
+                all_details.append({
+                    'value': netsuite_error_message,
+                    'type': detail['code'],
+                    'message': detail['message']
+                })
+                if not payment:
+                    Error.objects.update_or_create(
+                    workspace_id=expense_group.workspace_id,
+                    expense_group=expense_group,
+                    defaults={
+                        'type': 'NETSUITE_ERROR',
+                        'error_title': netsuite_error_message,
+                        'error_detail': detail['message'],
+                        'is_resolved': False
+                    }
+                )
+                task_log.detail = all_details
+
+                task_log.save()
+
+            except BulkError as exception:
+                logger.info(exception.response)
+                detail = exception.response
+                task_log.status = 'FAILED'
+                task_log.detail = detail
+
+                task_log.save()
+
+            except NetSuiteRateLimitError:
+                if not payment:
+                    Error.objects.update_or_create(
+                    workspace_id=expense_group.workspace_id,
+                    expense_group=expense_group,
+                    defaults={
+                        'type': 'NETSUITE_ERROR',
+                        'error_title': netsuite_error_message,
+                        'error_detail': f'Rate limit error, workspace_id - {expense_group.workspace_id}',
+                        'is_resolved': False
+                    }
+                )
+                logger.info('Rate limit error, workspace_id - %s', workspace_id if payment else expense_group.workspace_id)
+                task_log.status = 'FAILED'
+                task_log.detail = {
+                    'error': 'Rate limit error'
+                }
+
+                task_log.save()
+
+            except Exception:
+                error = traceback.format_exc()
+                task_log.detail = {
+                    'error': error
+                }
+                task_log.status = 'FATAL'
+                task_log.save()
+                __log_error(task_log)
+
+            if not payment and last_export is True:
+                update_last_export_details(expense_group.workspace_id)
+
+        return wrapper
+    return decorator