diff --git a/apps/mappings/signals.py b/apps/mappings/signals.py index 220049f8..c6fd04f5 100644 --- a/apps/mappings/signals.py +++ b/apps/mappings/signals.py @@ -5,7 +5,7 @@ from django.dispatch import receiver from django_q.tasks import async_task -from fyle_accounting_mappings.models import MappingSetting +from fyle_accounting_mappings.models import MappingSetting, EmployeeMapping, Mapping, CategoryMapping from apps.mappings.tasks import upload_attributes_to_fyle, schedule_cost_centers_creation,\ schedule_fyle_attributes_creation @@ -13,10 +13,38 @@ from apps.netsuite.helpers import schedule_payment_sync from apps.workspaces.models import Configuration from apps.workspaces.tasks import delete_cards_mapping_settings +from apps.tasks.models import Error from .models import GeneralMapping, SubsidiaryMapping from .tasks import schedule_auto_map_ccc_employees +@receiver(post_save, sender=Mapping) +def resolve_post_mapping_errors(sender, instance: Mapping, **kwargs): + """ + Resolve errors after mapping is created + """ + if instance.source_type == 'TAX_GROUP': + Error.objects.filter(expense_attribute_id=instance.source_id).update( + is_resolved=True + ) + +@receiver(post_save, sender=CategoryMapping) +def resolve_post_category_mapping_errors(sender, instance: Mapping, **kwargs): + """ + Resolve errors after mapping is created + """ + Error.objects.filter(expense_attribute_id=instance.source_category_id).update( + is_resolved=True + ) + +@receiver(post_save, sender=EmployeeMapping) +def resolve_post_employees_mapping_errors(sender, instance: Mapping, **kwargs): + """ + Resolve errors after mapping is created + """ + Error.objects.filter(expense_attribute_id=instance.source_employee_id).update( + is_resolved=True + ) @receiver(post_save, sender=SubsidiaryMapping) def run_post_subsidiary_mappings(sender, instance: SubsidiaryMapping, **kwargs): diff --git a/apps/mappings/tasks.py b/apps/mappings/tasks.py index f36b34d3..7f6964c9 100644 --- a/apps/mappings/tasks.py +++ b/apps/mappings/tasks.py @@ -13,13 +13,14 @@ from fyle.platform.exceptions import WrongParamsError, InvalidTokenError from fyle_accounting_mappings.models import Mapping, MappingSetting, ExpenseAttribute, DestinationAttribute,\ - CategoryMapping + CategoryMapping, EmployeeMapping from fyle_accounting_mappings.helpers import EmployeesAutoMappingHelper from fyle_integrations_platform_connector import PlatformConnector from apps.mappings.models import GeneralMapping from apps.netsuite.connector import NetSuiteConnector from apps.workspaces.models import NetSuiteCredentials, FyleCredential, Configuration, Workspace +from apps.tasks.models import Error from .exceptions import handle_exceptions from .constants import FYLE_EXPENSE_SYSTEM_FIELDS, DEFAULT_NETSUITE_IMPORT_TYPES @@ -27,6 +28,63 @@ logger = logging.getLogger(__name__) logger.level = logging.INFO +def get_mapped_attributes_ids(source_attribute_type: str, destination_attribute_type: str, errored_attribute_ids: List[int]): + + mapped_attribute_ids = [] + + if source_attribute_type == "TAX_GROUP": + mapped_attribute_ids: List[int] = Mapping.objects.filter( + source_id__in=errored_attribute_ids + ).values_list('source_id', flat=True) + + elif source_attribute_type == "EMPLOYEE": + params = { + 'source_employee_id__in': errored_attribute_ids, + } + + if destination_attribute_type == "EMPLOYEE": + params['destination_employee_id__isnull'] = False + else: + params['destination_vendor_id__isnull'] = False + mapped_attribute_ids: List[int] = EmployeeMapping.objects.filter( + **params + ).values_list('source_employee_id', flat=True) + + elif source_attribute_type == "CATEGORY": + params = { + 'source_category_id__in': errored_attribute_ids, + } + + if destination_attribute_type == 'EXPENSE_TYPE': + params['destination_expense_head_id__isnull'] = False + else: + params['destination_account_id__isnull'] = False + + mapped_attribute_ids: List[int] = CategoryMapping.objects.filter( + **params + ).values_list('source_category_id', flat=True) + + return mapped_attribute_ids + + +def resolve_expense_attribute_errors( + source_attribute_type: str, workspace_id: int, destination_attribute_type: str = None): + """ + Resolve Expense Attribute Errors + :return: None + """ + errored_attribute_ids: List[int] = Error.objects.filter( + is_resolved=False, + workspace_id=workspace_id, + type='{}_MAPPING'.format(source_attribute_type) + ).values_list('expense_attribute_id', flat=True) + + if errored_attribute_ids: + mapped_attribute_ids = get_mapped_attributes_ids(source_attribute_type, destination_attribute_type, errored_attribute_ids) + + if mapped_attribute_ids: + Error.objects.filter(expense_attribute_id__in=mapped_attribute_ids).update(is_resolved=True) + def remove_duplicates(ns_attributes: List[DestinationAttribute]): unique_attributes = [] @@ -467,6 +525,9 @@ def post_tax_groups(platform_connection: PlatformConnector, workspace_id: int): platform_connection.tax_groups.sync() Mapping.bulk_create_mappings(netsuite_attributes, 'TAX_GROUP', 'TAX_ITEM', workspace_id) + resolve_expense_attribute_errors( + source_attribute_type='TAX_GROUP', workspace_id=workspace_id + ) @handle_exceptions(task_name='Import Category to Fyle and Auto Create Mappings') def auto_create_category_mappings(workspace_id): @@ -505,6 +566,12 @@ def auto_create_category_mappings(workspace_id): if reimbursable_expenses_object == 'EXPENSE REPORT' and \ corporate_credit_card_expenses_object in ('BILL', 'JOURNAL ENTRY', 'CREDIT CARD CHARGE'): bulk_create_ccc_category_mappings(workspace_id) + + resolve_expense_attribute_errors( + source_attribute_type="CATEGORY", + destination_attribute_type=reimbursable_destination_type, + workspace_id=workspace_id + ) def auto_import_and_map_fyle_fields(workspace_id): @@ -657,6 +724,11 @@ def async_auto_map_employees(workspace_id: int): netsuite_connection.sync_vendors() EmployeesAutoMappingHelper(workspace_id, destination_type, employee_mapping_preference).reimburse_mapping() + resolve_expense_attribute_errors( + source_attribute_type="EMPLOYEE", + workspace_id=workspace_id, + destination_attribute_type=destination_type, + ) def schedule_auto_map_employees(employee_mapping_preference: str, workspace_id: int): diff --git a/apps/netsuite/actions.py b/apps/netsuite/actions.py new file mode 100644 index 00000000..2d0cc7aa --- /dev/null +++ b/apps/netsuite/actions.py @@ -0,0 +1,25 @@ +from apps.tasks.models import TaskLog +from apps.workspaces.models import LastExportDetail +from django.db.models import Q + + +def update_last_export_details(workspace_id): + last_export_detail = LastExportDetail.objects.get(workspace_id=workspace_id) + + failed_exports = TaskLog.objects.filter( + ~Q(type__in=['CREATING_VENDOR_PAYMENT','FETCHING_EXPENSES']), workspace_id=workspace_id, status__in=['FAILED', 'FATAL'] + ).count() + + successful_exports = TaskLog.objects.filter( + ~Q(type__in=['CREATING_VENDOR_PAYMENT', 'FETCHING_EXPENSES']), + workspace_id=workspace_id, + status='COMPLETE', + updated_at__gt=last_export_detail.last_exported_at + ).count() + + last_export_detail.failed_expense_groups_count = failed_exports + last_export_detail.successful_expense_groups_count = successful_exports + last_export_detail.total_expense_groups_count = failed_exports + successful_exports + last_export_detail.save() + + return last_export_detail diff --git a/apps/netsuite/exceptions.py b/apps/netsuite/exceptions.py new file mode 100644 index 00000000..21a2d4e6 --- /dev/null +++ b/apps/netsuite/exceptions.py @@ -0,0 +1,157 @@ +import logging +import json +import traceback + +from apps.fyle.models import ExpenseGroup +from apps.tasks.models import TaskLog, Error +from apps.workspaces.models import LastExportDetail, NetSuiteCredentials + +from netsuitesdk.internal.exceptions import NetSuiteRequestError +from netsuitesdk import NetSuiteRateLimitError, NetSuiteLoginError +from fyle_netsuite_api.exceptions import BulkError + +from .actions import update_last_export_details + +from django.db.models import Q + +logger = logging.getLogger(__name__) +logger.level = logging.INFO + +netsuite_error_message = 'NetSuite System Error' + +def __handle_netsuite_connection_error(expense_group: ExpenseGroup, task_log: TaskLog, workspace_id: int) -> None: + + if expense_group: + logger.info( + 'NetSuite Credentials not found for workspace_id %s / expense group %s', + expense_group.id, + expense_group.workspace_id + ) + else: + logger.info( + 'NetSuite Credentials not found for workspace_id %s', + workspace_id + ) + detail = { + 'message': 'NetSuite Account not connected' + } + + if expense_group: + Error.objects.update_or_create( + workspace_id=expense_group.workspace_id, + expense_group=expense_group, + defaults={ + 'type': 'NETSUITE_ERROR', + 'error_title': netsuite_error_message, + 'error_detail': detail['message'], + 'is_resolved': False + }) + + task_log.status = 'FAILED' + task_log.detail = detail + + task_log.save() + + +def __log_error(task_log: TaskLog) -> None: + logger.exception('Something unexpected happened workspace_id: %s %s', task_log.workspace_id, task_log.detail) + + +def handle_netsuite_exceptions(payment=False): + def decorator(func): + def wrapper(*args): + if payment: + entity_object = args[0] + workspace_id = args[1] + object_type = args[2] + expense_group = None + task_log, _ = TaskLog.objects.update_or_create( + workspace_id=workspace_id, + task_id='PAYMENT_{}'.format(entity_object['unique_id']), + defaults={ + 'status': 'IN_PROGRESS', + 'type': 'CREATING_VENDOR_PAYMENT' + } + ) + else: + expense_group = args[0] + workspace_id=expense_group.workspace_id + task_log_id = args[1] + task_log = TaskLog.objects.get(id=task_log_id) + last_export = args[2] + + try: + func(*args) + + except NetSuiteCredentials.DoesNotExist: + __handle_netsuite_connection_error(expense_group, task_log, workspace_id) + + except (NetSuiteRequestError, NetSuiteLoginError) as exception: + all_details = [] + logger.info({'error': exception}) + detail = json.dumps(exception.__dict__) + detail = json.loads(detail) + task_log.status = 'FAILED' + + all_details.append({ + 'value': netsuite_error_message, + 'type': detail['code'], + 'message': detail['message'] + }) + if not payment: + Error.objects.update_or_create( + workspace_id=expense_group.workspace_id, + expense_group=expense_group, + defaults={ + 'type': 'NETSUITE_ERROR', + 'error_title': netsuite_error_message, + 'error_detail': detail['message'], + 'is_resolved': False + } + ) + task_log.detail = all_details + + task_log.save() + + except BulkError as exception: + logger.info(exception.response) + detail = exception.response + task_log.status = 'FAILED' + task_log.detail = detail + + task_log.save() + + except NetSuiteRateLimitError: + if not payment: + Error.objects.update_or_create( + workspace_id=expense_group.workspace_id, + expense_group=expense_group, + defaults={ + 'type': 'NETSUITE_ERROR', + 'error_title': netsuite_error_message, + 'error_detail': f'Rate limit error, workspace_id - {expense_group.workspace_id}', + 'is_resolved': False + } + ) + logger.info('Rate limit error, workspace_id - %s', workspace_id if payment else expense_group.workspace_id) + task_log.status = 'FAILED' + task_log.detail = { + 'error': 'Rate limit error' + } + + task_log.save() + + except Exception: + error = traceback.format_exc() + task_log.detail = { + 'error': error + } + task_log.status = 'FATAL' + task_log.save() + __log_error(task_log) + + if not payment and last_export is True: + update_last_export_details(expense_group.workspace_id) + + return wrapper + return decorator diff --git a/apps/netsuite/tasks.py b/apps/netsuite/tasks.py index fe0977b1..e389c34c 100644 --- a/apps/netsuite/tasks.py +++ b/apps/netsuite/tasks.py @@ -9,6 +9,7 @@ from django.db import transaction from django.db.models import Q from django.utils.module_loading import import_string +from apps.netsuite.exceptions import handle_netsuite_exceptions from django_q.models import Schedule from django_q.tasks import Chain, async_task @@ -23,7 +24,7 @@ from apps.fyle.models import ExpenseGroup, Expense, Reimbursement from apps.mappings.models import GeneralMapping, SubsidiaryMapping -from apps.tasks.models import TaskLog +from apps.tasks.models import TaskLog, Error from apps.workspaces.models import LastExportDetail, NetSuiteCredentials, FyleCredential, Configuration, Workspace from .models import Bill, BillLineitem, ExpenseReport, ExpenseReportLineItem, JournalEntry, JournalEntryLineItem, \ @@ -71,27 +72,6 @@ } -def update_last_export_details(workspace_id): - last_export_detail = LastExportDetail.objects.get(workspace_id=workspace_id) - - failed_exports = TaskLog.objects.filter( - ~Q(type__in=['CREATING_VENDOR_PAYMENT','FETCHING_EXPENSES']), workspace_id=workspace_id, status__in=['FAILED', 'FATAL'] - ).count() - - successful_exports = TaskLog.objects.filter( - ~Q(type__in=['CREATING_VENDOR_PAYMENT', 'FETCHING_EXPENSES']), - workspace_id=workspace_id, - status='COMPLETE', - updated_at__gt=last_export_detail.last_exported_at - ).count() - - last_export_detail.failed_expense_groups_count = failed_exports - last_export_detail.successful_expense_groups_count = successful_exports - last_export_detail.total_expense_groups_count = failed_exports + successful_exports - last_export_detail.save() - - return last_export_detail - def load_attachments(netsuite_connection: NetSuiteConnector, expense: Expense, expense_group: ExpenseGroup): """ Get attachments from Fyle @@ -274,22 +254,7 @@ def create_or_update_employee_mapping(expense_group: ExpenseGroup, netsuite_conn except NetSuiteRequestError as exception: logger.info({'error': exception}) - - -def __handle_netsuite_connection_error(expense_group: ExpenseGroup, task_log: TaskLog) -> None: - logger.info( - 'NetSuite Credentials not found for workspace_id %s / expense group %s', - expense_group.id, - expense_group.workspace_id - ) - detail = { - 'expense_group_id': expense_group.id, - 'message': 'NetSuite Account not connected' - } - task_log.status = 'FAILED' - task_log.detail = detail - - task_log.save() + def construct_payload_and_update_export(expense_id_receipt_url_map: dict, task_log: TaskLog, workspace: Workspace, cluster_domain: str, netsuite_connection: NetSuiteConnector): @@ -422,6 +387,18 @@ def upload_attachments_and_update_export(expenses: List[Expense], task_log: Task ) +def resolve_errors_for_exported_expense_group(expense_group, workspace_id=None): + """ + Resolve errors for exported expense group + :param expense_group: Expense group + """ + if isinstance(expense_group, list): + Error.objects.filter(workspace_id=workspace_id, expense_group_id__in=expense_group, is_resolved=False).update(is_resolved=True) + else: + Error.objects.filter(workspace_id=expense_group.workspace_id, expense_group=expense_group, is_resolved=False).update(is_resolved=True) + + +@handle_netsuite_exceptions(payment=False) def create_bill(expense_group, task_log_id, last_export): task_log = TaskLog.objects.get(id=task_log_id) @@ -434,97 +411,46 @@ def create_bill(expense_group, task_log_id, last_export): configuration: Configuration = Configuration.objects.get(workspace_id=expense_group.workspace_id) general_mappings: GeneralMapping = GeneralMapping.objects.filter(workspace_id=expense_group.workspace_id).first() - try: - fyle_credentials = FyleCredential.objects.get(workspace_id=expense_group.workspace_id) - netsuite_credentials = NetSuiteCredentials.objects.get(workspace_id=expense_group.workspace_id) - - netsuite_connection = NetSuiteConnector(netsuite_credentials, expense_group.workspace_id) - - if expense_group.fund_source == 'PERSONAL' and configuration.auto_map_employees \ - and configuration.auto_create_destination_entity: - create_or_update_employee_mapping( - expense_group, netsuite_connection, configuration.auto_map_employees, - configuration.employee_field_mapping) - - if general_mappings and general_mappings.use_employee_department and expense_group.fund_source == 'CCC' \ - and configuration.auto_map_employees and configuration.auto_create_destination_entity: - create_or_update_employee_mapping( - expense_group, netsuite_connection, configuration.auto_map_employees, - configuration.employee_field_mapping) - - __validate_expense_group(expense_group, configuration) - with transaction.atomic(): - bill_object = Bill.create_bill(expense_group) - - bill_lineitems_objects = BillLineitem.create_bill_lineitems(expense_group, configuration) - - created_bill = netsuite_connection.post_bill(bill_object, bill_lineitems_objects) - - task_log.detail = created_bill - task_log.bill = bill_object - task_log.status = 'COMPLETE' - - task_log.save() + fyle_credentials = FyleCredential.objects.get(workspace_id=expense_group.workspace_id) + netsuite_credentials = NetSuiteCredentials.objects.get(workspace_id=expense_group.workspace_id) - expense_group.exported_at = datetime.now() - expense_group.response_logs = created_bill - expense_group.save() + netsuite_connection = NetSuiteConnector(netsuite_credentials, expense_group.workspace_id) - async_task( - 'apps.netsuite.tasks.upload_attachments_and_update_export', - expense_group.expenses.all(), task_log, fyle_credentials, expense_group.workspace_id - ) + if expense_group.fund_source == 'PERSONAL' and configuration.auto_map_employees \ + and configuration.auto_create_destination_entity: + create_or_update_employee_mapping( + expense_group, netsuite_connection, configuration.auto_map_employees, + configuration.employee_field_mapping) - except NetSuiteCredentials.DoesNotExist: - __handle_netsuite_connection_error(expense_group, task_log) + if general_mappings and general_mappings.use_employee_department and expense_group.fund_source == 'CCC' \ + and configuration.auto_map_employees and configuration.auto_create_destination_entity: + create_or_update_employee_mapping( + expense_group, netsuite_connection, configuration.auto_map_employees, + configuration.employee_field_mapping) - except (NetSuiteRequestError, NetSuiteLoginError) as exception: - all_details = [] - logger.info({'error': exception}) - detail = json.dumps(exception.__dict__) - detail = json.loads(detail) - task_log.status = 'FAILED' + __validate_expense_group(expense_group, configuration) + with transaction.atomic(): + bill_object = Bill.create_bill(expense_group) - all_details.append({ - 'expense_group_id': expense_group.id, - 'value': netsuite_error_message, - 'type': detail['code'], - 'message': detail['message'] - }) - task_log.detail = all_details + bill_lineitems_objects = BillLineitem.create_bill_lineitems(expense_group, configuration) - task_log.save() + created_bill = netsuite_connection.post_bill(bill_object, bill_lineitems_objects) - except BulkError as exception: - logger.info(exception.response) - detail = exception.response - task_log.status = 'FAILED' - task_log.detail = detail + task_log.detail = created_bill + task_log.bill = bill_object + task_log.status = 'COMPLETE' task_log.save() - except NetSuiteRateLimitError: - logger.info('Rate limit error, workspace_id - %s', expense_group.workspace_id) - task_log.status = 'FAILED' - task_log.detail = { - 'error': 'Rate limit error' - } - - task_log.save() + expense_group.exported_at = datetime.now() + expense_group.response_logs = created_bill + expense_group.save() + resolve_errors_for_exported_expense_group(expense_group) - except Exception: - error = traceback.format_exc() - task_log.detail = { - 'error': error - } - task_log.status = 'FATAL' task_log.save() - __log_error(task_log) - - if last_export: - update_last_export_details(expense_group.workspace_id) +@handle_netsuite_exceptions(payment=False) def create_credit_card_charge(expense_group, task_log_id, last_export): task_log = TaskLog.objects.get(id=task_log_id) @@ -537,109 +463,61 @@ def create_credit_card_charge(expense_group, task_log_id, last_export): configuration = Configuration.objects.get(workspace_id=expense_group.workspace_id) general_mappings: GeneralMapping = GeneralMapping.objects.filter(workspace_id=expense_group.workspace_id).first() - try: - netsuite_credentials = NetSuiteCredentials.objects.get(workspace_id=expense_group.workspace_id) - - netsuite_connection = NetSuiteConnector(netsuite_credentials, expense_group.workspace_id) - - if general_mappings and general_mappings.use_employee_department and expense_group.fund_source == 'CCC' \ - and configuration.auto_map_employees and configuration.auto_create_destination_entity: - create_or_update_employee_mapping( - expense_group, netsuite_connection, configuration.auto_map_employees, - configuration.employee_field_mapping) - - merchant = expense_group.expenses.first().vendor - auto_create_merchants = configuration.auto_create_merchants - get_or_create_credit_card_vendor(expense_group, merchant, auto_create_merchants) - - __validate_expense_group(expense_group, configuration) - with transaction.atomic(): - credit_card_charge_object = CreditCardCharge.create_credit_card_charge(expense_group) - - credit_card_charge_lineitems_object = CreditCardChargeLineItem.create_credit_card_charge_lineitem( - expense_group, configuration - ) - attachment_links = {} - - expense = expense_group.expenses.first() - refund = False - if expense.amount < 0: - refund = True + netsuite_credentials = NetSuiteCredentials.objects.get(workspace_id=expense_group.workspace_id) - attachment_link = load_attachments(netsuite_connection, expense, expense_group) + netsuite_connection = NetSuiteConnector(netsuite_credentials, expense_group.workspace_id) - if attachment_link: - attachment_links[expense.expense_id] = attachment_link + if general_mappings and general_mappings.use_employee_department and expense_group.fund_source == 'CCC' \ + and configuration.auto_map_employees and configuration.auto_create_destination_entity: + create_or_update_employee_mapping( + expense_group, netsuite_connection, configuration.auto_map_employees, + configuration.employee_field_mapping) - created_credit_card_charge = netsuite_connection.post_credit_card_charge( - credit_card_charge_object, credit_card_charge_lineitems_object, attachment_links, refund - ) + merchant = expense_group.expenses.first().vendor + auto_create_merchants = configuration.auto_create_merchants + get_or_create_credit_card_vendor(expense_group, merchant, auto_create_merchants) - if refund: - created_credit_card_charge['type'] = 'chargeCardRefund' - else: - created_credit_card_charge['type'] = 'chargeCard' + __validate_expense_group(expense_group, configuration) + with transaction.atomic(): + credit_card_charge_object = CreditCardCharge.create_credit_card_charge(expense_group) - task_log.detail = created_credit_card_charge - task_log.credit_card_charge = credit_card_charge_object - task_log.status = 'COMPLETE' - - task_log.save() + credit_card_charge_lineitems_object = CreditCardChargeLineItem.create_credit_card_charge_lineitem( + expense_group, configuration + ) + attachment_links = {} - expense_group.exported_at = datetime.now() - expense_group.response_logs = created_credit_card_charge - expense_group.save() + expense = expense_group.expenses.first() + refund = False + if expense.amount < 0: + refund = True - except NetSuiteCredentials.DoesNotExist: - __handle_netsuite_connection_error(expense_group, task_log) + attachment_link = load_attachments(netsuite_connection, expense, expense_group) - except (NetSuiteRequestError, NetSuiteLoginError) as exception: - all_details = [] - logger.info({'error': exception}) - detail = json.dumps(exception.__dict__) - detail = json.loads(detail) - task_log.status = 'FAILED' + if attachment_link: + attachment_links[expense.expense_id] = attachment_link - all_details.append({ - 'expense_group_id': expense_group.id, - 'value': netsuite_error_message, - 'type': detail['code'], - 'message': detail['message'] - }) - task_log.detail = all_details + created_credit_card_charge = netsuite_connection.post_credit_card_charge( + credit_card_charge_object, credit_card_charge_lineitems_object, attachment_links, refund + ) - task_log.save() + if refund: + created_credit_card_charge['type'] = 'chargeCardRefund' + else: + created_credit_card_charge['type'] = 'chargeCard' - except BulkError as exception: - logger.info(exception.response) - detail = exception.response - task_log.status = 'FAILED' - task_log.detail = detail + task_log.detail = created_credit_card_charge + task_log.credit_card_charge = credit_card_charge_object + task_log.status = 'COMPLETE' task_log.save() - except NetSuiteRateLimitError: - logger.info('Rate limit error, workspace_id - %s', expense_group.workspace_id) - task_log.status = 'FAILED' - task_log.detail = { - 'error': 'Rate limit error' - } - - task_log.save() - - except Exception: - error = traceback.format_exc() - task_log.detail = { - 'error': error - } - task_log.status = 'FATAL' - task_log.save() - __log_error(task_log) - - if last_export: - update_last_export_details(expense_group.workspace_id) + expense_group.exported_at = datetime.now() + expense_group.response_logs = created_credit_card_charge + expense_group.save() + resolve_errors_for_exported_expense_group(expense_group) +@handle_netsuite_exceptions(payment=False) def create_expense_report(expense_group, task_log_id, last_export): task_log = TaskLog.objects.get(id=task_log_id) @@ -651,92 +529,42 @@ def create_expense_report(expense_group, task_log_id, last_export): configuration = Configuration.objects.get(workspace_id=expense_group.workspace_id) - try: - fyle_credentials = FyleCredential.objects.get(workspace_id=expense_group.workspace_id) - netsuite_credentials = NetSuiteCredentials.objects.get(workspace_id=expense_group.workspace_id) - netsuite_connection = NetSuiteConnector(netsuite_credentials, expense_group.workspace_id) - - if configuration.auto_map_employees and configuration.auto_create_destination_entity: - create_or_update_employee_mapping( - expense_group, netsuite_connection, configuration.auto_map_employees, - configuration.employee_field_mapping) - - __validate_expense_group(expense_group, configuration) - with transaction.atomic(): - expense_report_object = ExpenseReport.create_expense_report(expense_group) - - expense_report_lineitems_objects = ExpenseReportLineItem.create_expense_report_lineitems( - expense_group, configuration - ) - - created_expense_report = netsuite_connection.post_expense_report( - expense_report_object, expense_report_lineitems_objects - ) - - task_log.detail = created_expense_report - task_log.expense_report = expense_report_object - task_log.status = 'COMPLETE' - - task_log.save() - - expense_group.exported_at = datetime.now() - expense_group.response_logs = created_expense_report - expense_group.save() - - async_task( - 'apps.netsuite.tasks.upload_attachments_and_update_export', - expense_group.expenses.all(), task_log, fyle_credentials, expense_group.workspace_id - ) + fyle_credentials = FyleCredential.objects.get(workspace_id=expense_group.workspace_id) + netsuite_credentials = NetSuiteCredentials.objects.get(workspace_id=expense_group.workspace_id) + netsuite_connection = NetSuiteConnector(netsuite_credentials, expense_group.workspace_id) - except NetSuiteCredentials.DoesNotExist: - __handle_netsuite_connection_error(expense_group, task_log) + if configuration.auto_map_employees and configuration.auto_create_destination_entity: + create_or_update_employee_mapping( + expense_group, netsuite_connection, configuration.auto_map_employees, + configuration.employee_field_mapping) - except (NetSuiteRequestError, NetSuiteLoginError) as exception: - all_details = [] - logger.info({'error': exception}) - detail = json.dumps(exception.__dict__) - detail = json.loads(detail) - task_log.status = 'FAILED' + __validate_expense_group(expense_group, configuration) + with transaction.atomic(): + expense_report_object = ExpenseReport.create_expense_report(expense_group) - all_details.append({ - 'expense_group_id': expense_group.id, - 'value': netsuite_error_message, - 'type': detail['code'], - 'message': detail['message'] - }) - task_log.detail = all_details + expense_report_lineitems_objects = ExpenseReportLineItem.create_expense_report_lineitems( + expense_group, configuration + ) - task_log.save() + created_expense_report = netsuite_connection.post_expense_report( + expense_report_object, expense_report_lineitems_objects + ) - except BulkError as exception: - logger.info(exception.response) - detail = exception.response - task_log.status = 'FAILED' - task_log.detail = detail + task_log.detail = created_expense_report + task_log.expense_report = expense_report_object + task_log.status = 'COMPLETE' task_log.save() - except NetSuiteRateLimitError: - logger.info('Rate limit error, workspace_id - %s', expense_group.workspace_id) - task_log.status = 'FAILED' - task_log.detail = { - 'error': 'Rate limit error' - } + expense_group.exported_at = datetime.now() + expense_group.response_logs = created_expense_report + expense_group.save() + resolve_errors_for_exported_expense_group(expense_group) task_log.save() - except Exception: - error = traceback.format_exc() - task_log.detail = { - 'error': error - } - task_log.status = 'FATAL' - task_log.save() - __log_error(task_log) - - if last_export: - update_last_export_details(expense_group.workspace_id) +@handle_netsuite_exceptions(payment=False) def create_journal_entry(expense_group, task_log_id, last_export): task_log = TaskLog.objects.get(id=task_log_id) @@ -748,91 +576,40 @@ def create_journal_entry(expense_group, task_log_id, last_export): configuration = Configuration.objects.get(workspace_id=expense_group.workspace_id) - try: - fyle_credentials = FyleCredential.objects.get(workspace_id=expense_group.workspace_id) - netsuite_credentials = NetSuiteCredentials.objects.get(workspace_id=expense_group.workspace_id) - - netsuite_connection = NetSuiteConnector(netsuite_credentials, expense_group.workspace_id) - - if configuration.auto_map_employees and configuration.auto_create_destination_entity: - create_or_update_employee_mapping( - expense_group, netsuite_connection, configuration.auto_map_employees, - configuration.employee_field_mapping) - __validate_expense_group(expense_group, configuration) - with transaction.atomic(): - journal_entry_object = JournalEntry.create_journal_entry(expense_group) - - journal_entry_lineitems_objects = JournalEntryLineItem.create_journal_entry_lineitems( - expense_group, configuration - ) - - created_journal_entry = netsuite_connection.post_journal_entry( - journal_entry_object, journal_entry_lineitems_objects - ) - - task_log.detail = created_journal_entry - task_log.journal_entry = journal_entry_object - task_log.status = 'COMPLETE' - - task_log.save() - expense_group.exported_at = datetime.now() - expense_group.response_logs = created_journal_entry - expense_group.save() - - async_task( - 'apps.netsuite.tasks.upload_attachments_and_update_export', - expense_group.expenses.all(), task_log, fyle_credentials, expense_group.workspace_id - ) + fyle_credentials = FyleCredential.objects.get(workspace_id=expense_group.workspace_id) + netsuite_credentials = NetSuiteCredentials.objects.get(workspace_id=expense_group.workspace_id) - except NetSuiteCredentials.DoesNotExist: - __handle_netsuite_connection_error(expense_group, task_log) + netsuite_connection = NetSuiteConnector(netsuite_credentials, expense_group.workspace_id) - except (NetSuiteRequestError, NetSuiteLoginError) as exception: - all_details = [] - logger.info({'error': exception}) - detail = json.dumps(exception.__dict__) - detail = json.loads(detail) - task_log.status = 'FAILED' + if configuration.auto_map_employees and configuration.auto_create_destination_entity: + create_or_update_employee_mapping( + expense_group, netsuite_connection, configuration.auto_map_employees, + configuration.employee_field_mapping) + __validate_expense_group(expense_group, configuration) + with transaction.atomic(): + journal_entry_object = JournalEntry.create_journal_entry(expense_group) - all_details.append({ - 'expense_group_id': expense_group.id, - 'value': netsuite_error_message, - 'type': detail['code'], - 'message': detail['message'] - }) - task_log.detail = all_details + journal_entry_lineitems_objects = JournalEntryLineItem.create_journal_entry_lineitems( + expense_group, configuration + ) - task_log.save() + created_journal_entry = netsuite_connection.post_journal_entry( + journal_entry_object, journal_entry_lineitems_objects + ) - except BulkError as exception: - logger.info(exception.response) - detail = exception.response - task_log.status = 'FAILED' - task_log.detail = detail + task_log.detail = created_journal_entry + task_log.journal_entry = journal_entry_object + task_log.status = 'COMPLETE' task_log.save() - except NetSuiteRateLimitError: - logger.info('Rate limit error, workspace_id - %s', expense_group.workspace_id) - task_log.status = 'FAILED' - task_log.detail = { - 'error': 'Rate limit error' - } - - task_log.save() + expense_group.exported_at = datetime.now() + expense_group.response_logs = created_journal_entry + expense_group.save() + resolve_errors_for_exported_expense_group(expense_group) - except Exception: - error = traceback.format_exc() - task_log.detail = { - 'error': error - } - task_log.status = 'FATAL' task_log.save() - __log_error(task_log) - - if last_export: - update_last_export_details(expense_group.workspace_id) def __validate_general_mapping(expense_group: ExpenseGroup, configuration: Configuration) -> List[BulkError]: @@ -969,6 +746,18 @@ def __validate_tax_group_mapping(expense_group: ExpenseGroup, configuration: Con 'message': 'Tax Group Mapping not found' }) + if tax_group: + Error.objects.update_or_create( + workspace_id=tax_group.workspace_id, + expense_attribute=tax_group, + defaults={ + 'type': 'TAX_MAPPING', + 'error_title': tax_group.value, + 'error_detail': 'Tax mapping is missing', + 'is_resolved': False + } + ) + row = row + 1 return bulk_errors @@ -1046,6 +835,18 @@ def __validate_employee_mapping(expense_group: ExpenseGroup, configuration: Conf 'message': 'Employee mapping not found' }) + if employee: + Error.objects.update_or_create( + workspace_id=expense_group.workspace_id, + expense_attribute=employee, + defaults={ + 'type': 'EMPLOYEE_MAPPING', + 'error_title': employee.value, + 'error_detail': 'Employee mapping is missing', + 'is_resolved': False + } + ) + return bulk_errors @@ -1063,6 +864,12 @@ def __validate_category_mapping(expense_group: ExpenseGroup, configuration: Conf workspace_id=expense_group.workspace_id ).first() + category_attribute = ExpenseAttribute.objects.filter( + value=category, + workspace_id=expense_group.workspace_id, + attribute_type='CATEGORY' + ).first() + if category_mapping: if expense_group.fund_source == 'PERSONAL': if configuration.reimbursable_expenses_object == 'EXPENSE REPORT': @@ -1084,6 +891,19 @@ def __validate_category_mapping(expense_group: ExpenseGroup, configuration: Conf 'message': 'Category Mapping Not Found' }) + if category_attribute: + Error.objects.update_or_create( + workspace_id=expense_group.workspace_id, + expense_attribute=category_attribute, + defaults={ + 'type': 'CATEGORY_MAPPING', + 'error_title': category_attribute.value, + 'error_detail': 'Category mapping is missing', + 'is_resolved': False + } + ) + + row = row + 1 return bulk_errors @@ -1371,7 +1191,7 @@ def create_netsuite_payment_objects(netsuite_objects, object_type, workspace_id) return netsuite_payment_objects - +@handle_netsuite_exceptions(payment=True) def process_vendor_payment(entity_object, workspace_id, object_type): task_log, _ = TaskLog.objects.update_or_create( workspace_id=workspace_id, @@ -1382,93 +1202,48 @@ def process_vendor_payment(entity_object, workspace_id, object_type): } ) - try: - with transaction.atomic(): - - vendor_payment_object = VendorPayment.create_vendor_payment( - workspace_id, entity_object - ) - - vendor_payment_lineitems = VendorPaymentLineitem.create_vendor_payment_lineitems( - entity_object['line'], vendor_payment_object - ) - - netsuite_credentials = NetSuiteCredentials.objects.get(workspace_id=workspace_id) - netsuite_connection = NetSuiteConnector(netsuite_credentials, workspace_id) - - first_object_id = vendor_payment_lineitems[0].doc_id - if object_type == 'BILL': - first_object = netsuite_connection.get_bill(first_object_id) - else: - first_object = netsuite_connection.get_expense_report(first_object_id) - created_vendor_payment = netsuite_connection.post_vendor_payment( - vendor_payment_object, vendor_payment_lineitems, first_object - ) - - lines = entity_object['line'] - expense_group_ids = [line['expense_group'].id for line in lines] + with transaction.atomic(): - if object_type == 'BILL': - paid_objects = Bill.objects.filter(expense_group_id__in=expense_group_ids).all() - - else: - paid_objects = ExpenseReport.objects.filter(expense_group_id__in=expense_group_ids).all() + vendor_payment_object = VendorPayment.create_vendor_payment( + workspace_id, entity_object + ) - for paid_object in paid_objects: - paid_object.payment_synced = True - paid_object.paid_on_netsuite = True - paid_object.save() + vendor_payment_lineitems = VendorPaymentLineitem.create_vendor_payment_lineitems( + entity_object['line'], vendor_payment_object + ) - task_log.detail = created_vendor_payment - task_log.vendor_payment = vendor_payment_object - task_log.status = 'COMPLETE' + netsuite_credentials = NetSuiteCredentials.objects.get(workspace_id=workspace_id) + netsuite_connection = NetSuiteConnector(netsuite_credentials, workspace_id) - task_log.save() - except NetSuiteCredentials.DoesNotExist: - logger.info( - 'NetSuite Credentials not found for workspace_id %s', - workspace_id + first_object_id = vendor_payment_lineitems[0].doc_id + if object_type == 'BILL': + first_object = netsuite_connection.get_bill(first_object_id) + else: + first_object = netsuite_connection.get_expense_report(first_object_id) + created_vendor_payment = netsuite_connection.post_vendor_payment( + vendor_payment_object, vendor_payment_lineitems, first_object ) - detail = { - 'message': 'NetSuite Account not connected' - } - task_log.status = 'FAILED' - task_log.detail = detail - task_log.save() + lines = entity_object['line'] + expense_group_ids = [line['expense_group'].id for line in lines] - except (NetSuiteRequestError, NetSuiteLoginError) as exception: - all_details = [] - logger.info({'error': exception}) - detail = json.dumps(exception.__dict__) - detail = json.loads(detail) - task_log.status = 'FAILED' - - all_details.append({ - 'value': netsuite_error_message, - 'type': detail['code'], - 'message': detail['message'] - }) - task_log.detail = all_details - - task_log.save() + if object_type == 'BILL': + paid_objects = Bill.objects.filter(expense_group_id__in=expense_group_ids).all() - except NetSuiteRateLimitError: - logger.info('Rate limit error, workspace_id - %s', workspace_id) - task_log.status = 'FAILED' - task_log.detail = { - 'error': 'Rate limit error' - } + else: + paid_objects = ExpenseReport.objects.filter(expense_group_id__in=expense_group_ids).all() - task_log.save() + for paid_object in paid_objects: + paid_object.payment_synced = True + paid_object.paid_on_netsuite = True + paid_object.save() - except BulkError as exception: - logger.info(exception.response) - detail = exception.response - task_log.status = 'FAILED' - task_log.detail = detail + task_log.detail = created_vendor_payment + task_log.vendor_payment = vendor_payment_object + task_log.status = 'COMPLETE' task_log.save() + resolve_errors_for_exported_expense_group(expense_group_ids, workspace_id) def create_vendor_payment(workspace_id): diff --git a/tests/test_mappings/test_signals.py b/tests/test_mappings/test_signals.py index 1f33c019..efccc420 100644 --- a/tests/test_mappings/test_signals.py +++ b/tests/test_mappings/test_signals.py @@ -2,7 +2,96 @@ from django_q.models import Schedule from apps.workspaces.models import Configuration, Workspace from apps.mappings.models import GeneralMapping -from fyle_accounting_mappings.models import MappingSetting, ExpenseAttribute +from apps.tasks.models import Error +from fyle_accounting_mappings.models import MappingSetting, ExpenseAttribute, EmployeeMapping, CategoryMapping, Mapping + + +def test_resolve_post_mapping_errors(access_token): + tax_group = ExpenseAttribute.objects.filter( + value='GST: NCF-AU @0.0%', + workspace_id=1, + attribute_type='TAX_GROUP' + ).first() + + Error.objects.update_or_create( + workspace_id=1, + expense_attribute=tax_group, + defaults={ + 'type': 'TAX_GROUP_MAPPING', + 'error_title': tax_group.value, + 'error_detail': 'Tax group mapping is missing', + 'is_resolved': False + } + ) + + mapping = Mapping( + source_type='TAX_GROUP', + destination_type='TAX_DETAIL', + # source__value=source_value, + source_id=1642, + destination_id=1019, + workspace_id=1 + ) + mapping.save() + error = Error.objects.filter(expense_attribute_id=mapping.source_id).first() + + assert error.is_resolved == True + +@pytest.mark.django_db() +def test_resolve_post_category_mapping_errors(access_token): + source_category = ExpenseAttribute.objects.filter( + id=96, + workspace_id=1, + attribute_type='CATEGORY' + ).first() + + Error.objects.update_or_create( + workspace_id=1, + expense_attribute=source_category, + defaults={ + 'type': 'CATEGORY_MAPPING', + 'error_title': source_category.value, + 'error_detail': 'Category mapping is missing', + 'is_resolved': False + } + ) + category_mapping, _ = CategoryMapping.objects.update_or_create( + source_category_id=96, + destination_account_id=791, + destination_expense_head_id=791, + workspace_id=1 + ) + + error = Error.objects.filter(expense_attribute_id=category_mapping.source_category_id).first() + assert error.is_resolved == True + +@pytest.mark.django_db() +def test_resolve_post_employees_mapping_errors(access_token): + source_employee = ExpenseAttribute.objects.filter( + value='approver1@fyleforgotham.in', + workspace_id=1, + attribute_type='EMPLOYEE' + ).first() + + Error.objects.update_or_create( + workspace_id=1, + expense_attribute=source_employee, + defaults={ + 'type': 'EMPLOYEE_MAPPING', + 'error_title': source_employee.value, + 'error_detail': 'Employee mapping is missing', + 'is_resolved': False + } + ) + employee_mapping, _ = EmployeeMapping.objects.update_or_create( + source_employee_id=1, + destination_employee_id=719, + workspace_id=1 + ) + + error = Error.objects.filter(expense_attribute_id=employee_mapping.source_employee_id).first() + + assert error.is_resolved == True @pytest.mark.django_db() def test_run_post_mapping_settings_triggers(access_token): diff --git a/tests/test_mappings/test_tasks.py b/tests/test_mappings/test_tasks.py index cb8f4d6f..c2c56a67 100644 --- a/tests/test_mappings/test_tasks.py +++ b/tests/test_mappings/test_tasks.py @@ -1,4 +1,5 @@ import logging +from apps.fyle.models import ExpenseGroup import pytest from unittest import mock from django_q.models import Schedule @@ -17,6 +18,66 @@ logger = logging.getLogger(__name__) logger.level = logging.INFO + +def test_resolve_expense_attribute_errors(db): + workspace_id = 1 + expense_group = ExpenseGroup.objects.get(id=1) + + employee_attribute = ExpenseAttribute.objects.filter( + value=expense_group.description.get('employee_email'), + workspace_id=expense_group.workspace_id, + attribute_type='EMPLOYEE' + ).first() + + error, _ = Error.objects.update_or_create( + workspace_id=expense_group.workspace_id, + expense_attribute=employee_attribute, + defaults={ + 'type': 'EMPLOYEE_MAPPING', + 'error_title': employee_attribute.value, + 'error_detail': 'Employee mapping is missing', + 'is_resolved': False + } + ) + + resolve_expense_attribute_errors('EMPLOYEE', workspace_id, 'EMPLOYEE') + assert Error.objects.get(id=error.id).is_resolved == True + + error, _ = Error.objects.update_or_create( + workspace_id=expense_group.workspace_id, + expense_attribute=employee_attribute, + defaults={ + 'type': 'EMPLOYEE_MAPPING', + 'error_title': employee_attribute.value, + 'error_detail': 'Employee mapping is missing', + 'is_resolved': False + } + ) + + resolve_expense_attribute_errors('EMPLOYEE', workspace_id, 'VENDOR') + assert Error.objects.get(id=error.id).is_resolved == True + + source_category = ExpenseAttribute.objects.filter( + id=34, + workspace_id=1, + attribute_type='CATEGORY' + ).first() + + error, _ = Error.objects.update_or_create( + workspace_id=1, + expense_attribute=source_category, + defaults={ + 'type': 'CATEGORY_MAPPING', + 'error_title': source_category.value, + 'error_detail': 'Category mapping is missing', + 'is_resolved': False + } + ) + + resolve_expense_attribute_errors('CATEGORY', workspace_id, 'ACCOUNT') + assert Error.objects.get(id=error.id).is_resolved == True + + def test_disable_category_for_items_mapping(db ,mocker): workspace_id = 49 configuration = Configuration.objects.filter(workspace_id=workspace_id).first() diff --git a/tests/test_netsuite/test_tasks.py b/tests/test_netsuite/test_tasks.py index c095995b..38f62c6a 100644 --- a/tests/test_netsuite/test_tasks.py +++ b/tests/test_netsuite/test_tasks.py @@ -15,8 +15,9 @@ from apps.workspaces.models import Configuration, LastExportDetail, NetSuiteCredentials, FyleCredential from apps.tasks.models import TaskLog from apps.netsuite.tasks import __validate_general_mapping, __validate_subsidiary_mapping, check_netsuite_object_status, create_credit_card_charge, create_journal_entry, create_or_update_employee_mapping, create_vendor_payment, get_all_internal_ids, \ - get_or_create_credit_card_vendor, create_bill, create_expense_report, load_attachments, __handle_netsuite_connection_error, process_reimbursements, process_vendor_payment, schedule_bills_creation, schedule_credit_card_charge_creation, schedule_expense_reports_creation, schedule_journal_entry_creation, schedule_netsuite_objects_status_sync, schedule_reimbursements_sync, schedule_vendor_payment_creation, \ + get_or_create_credit_card_vendor, create_bill, create_expense_report, load_attachments, process_reimbursements, process_vendor_payment, schedule_bills_creation, schedule_credit_card_charge_creation, schedule_expense_reports_creation, schedule_journal_entry_creation, schedule_netsuite_objects_status_sync, schedule_reimbursements_sync, schedule_vendor_payment_creation, \ __validate_tax_group_mapping, check_expenses_reimbursement_status, __validate_expense_group, upload_attachments_and_update_export +from apps.netsuite.exceptions import __handle_netsuite_connection_error from apps.mappings.models import GeneralMapping, SubsidiaryMapping from fyle_accounting_mappings.models import DestinationAttribute, EmployeeMapping, CategoryMapping, ExpenseAttribute, Mapping from .fixtures import data @@ -958,7 +959,7 @@ def test_handle_netsuite_connection_error(db): workspace_id=1 ) - __handle_netsuite_connection_error(expense_group, task_log) + __handle_netsuite_connection_error(expense_group, task_log, workspace_id=1) task_log = TaskLog.objects.filter(workspace_id=1).last()