diff --git a/apps/quickbooks_online/queue.py b/apps/quickbooks_online/queue.py index fdd1b14d..82b31b8f 100644 --- a/apps/quickbooks_online/queue.py +++ b/apps/quickbooks_online/queue.py @@ -1,4 +1,5 @@ -from datetime import datetime, timedelta +import logging +from datetime import datetime, timedelta, timezone from typing import List from django.db.models import Q @@ -6,15 +7,30 @@ from django_q.tasks import Chain, async_task from apps.fyle.models import Expense, ExpenseGroup -from apps.tasks.models import TaskLog +from apps.tasks.models import TaskLog, Error from apps.workspaces.models import FyleCredential, WorkspaceGeneralSettings +logger = logging.getLogger(__name__) +logger.level = logging.INFO + + def async_run_post_configration_triggers(workspace_general_settings: WorkspaceGeneralSettings): async_task('apps.quickbooks_online.tasks.async_sync_accounts', int(workspace_general_settings.workspace_id), q_options={'cluster': 'import'}) -def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str): +def validate_failing_export(is_auto_export: bool, interval_hours: int, error: Error): + """ + Validate failing export + :param is_auto_export: Is auto export + :param interval_hours: Interval hours + :param error: Error + """ + # If auto export is enabled and interval hours is set and error repetition count is greater than 100, export only once a day + return is_auto_export and interval_hours and error and error.repetition_count > 100 and datetime.now().replace(tzinfo=timezone.utc) - error.updated_at <= timedelta(hours=24) + + +def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str, interval_hours: int): """ Schedule bills creation :param expense_group_ids: List of expense group ids @@ -26,10 +42,17 @@ def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str], is_ if expense_group_ids: expense_groups = ExpenseGroup.objects.filter(Q(tasklog__id__isnull=True) | ~Q(tasklog__status__in=['IN_PROGRESS', 'COMPLETE']), workspace_id=workspace_id, id__in=expense_group_ids, bill__id__isnull=True, exported_at__isnull=True).all() + errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, expense_group_id__in=expense_group_ids).all() + chain_tasks = [] in_progress_expenses = [] for index, expense_group in enumerate(expense_groups): + error = errors.filter(workspace_id=workspace_id, expense_group=expense_group, is_resolved=False).first() + skip_export = validate_failing_export(is_auto_export, interval_hours, error) + if skip_export: + logger.info('Skipping expense group %s as it has %s errors', expense_group.id, error.repetition_count) + continue task_log, _ = TaskLog.objects.get_or_create(workspace_id=expense_group.workspace_id, expense_group=expense_group, defaults={'status': 'ENQUEUED', 'type': 'CREATING_BILL'}) if task_log.status not in ['IN_PROGRESS', 'ENQUEUED']: task_log.type = 'CREATING_BILL' @@ -79,7 +102,7 @@ def __create_chain_and_run(fyle_credentials: FyleCredential, in_progress_expense chain.run() -def schedule_cheques_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str): +def schedule_cheques_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str, interval_hours: int): """ Schedule cheque creation :param expense_group_ids: List of expense group ids @@ -91,10 +114,17 @@ def schedule_cheques_creation(workspace_id: int, expense_group_ids: List[str], i if expense_group_ids: expense_groups = ExpenseGroup.objects.filter(Q(tasklog__id__isnull=True) | ~Q(tasklog__status__in=['IN_PROGRESS', 'COMPLETE']), workspace_id=workspace_id, id__in=expense_group_ids, cheque__id__isnull=True, exported_at__isnull=True).all() + errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, expense_group_id__in=expense_group_ids).all() + chain_tasks = [] in_progress_expenses = [] for index, expense_group in enumerate(expense_groups): + error = errors.filter(workspace_id=workspace_id, expense_group=expense_group, is_resolved=False).first() + skip_export = validate_failing_export(is_auto_export, interval_hours, error) + if skip_export: + logger.info('Skipping expense group %s as it has %s errors', expense_group.id, error.repetition_count) + continue task_log, _ = TaskLog.objects.get_or_create(workspace_id=expense_group.workspace_id, expense_group=expense_group, defaults={'status': 'ENQUEUED', 'type': 'CREATING_CHECK'}) if task_log.status not in ['IN_PROGRESS', 'ENQUEUED']: task_log.type = 'CREATING_CHECK' @@ -121,7 +151,7 @@ def schedule_cheques_creation(workspace_id: int, expense_group_ids: List[str], i __create_chain_and_run(fyle_credentials, in_progress_expenses, workspace_id, chain_tasks, fund_source) -def schedule_journal_entry_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str): +def schedule_journal_entry_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str, interval_hours: int): """ Schedule journal_entry creation :param expense_group_ids: List of expense group ids @@ -133,10 +163,17 @@ def schedule_journal_entry_creation(workspace_id: int, expense_group_ids: List[s Q(tasklog__id__isnull=True) | ~Q(tasklog__status__in=['IN_PROGRESS', 'COMPLETE']), workspace_id=workspace_id, id__in=expense_group_ids, journalentry__id__isnull=True, exported_at__isnull=True ).all() + errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, expense_group_id__in=expense_group_ids).all() + chain_tasks = [] in_progress_expenses = [] for index, expense_group in enumerate(expense_groups): + error = errors.filter(workspace_id=workspace_id, expense_group=expense_group, is_resolved=False).first() + skip_export = validate_failing_export(is_auto_export, interval_hours, error) + if skip_export: + logger.info('Skipping expense group %s as it has %s errors', expense_group.id, error.repetition_count) + continue task_log, _ = TaskLog.objects.get_or_create(workspace_id=expense_group.workspace_id, expense_group=expense_group, defaults={'status': 'ENQUEUED', 'type': 'CREATING_JOURNAL_ENTRY'}) if task_log.status not in ['IN_PROGRESS', 'ENQUEUED']: task_log.type = 'CREATING_JOURNAL_ENTRY' @@ -162,7 +199,7 @@ def schedule_journal_entry_creation(workspace_id: int, expense_group_ids: List[s __create_chain_and_run(fyle_credentials, in_progress_expenses, workspace_id, chain_tasks, fund_source) -def schedule_credit_card_purchase_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str): +def schedule_credit_card_purchase_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str, interval_hours: int): """ Schedule credit card purchase creation :param expense_group_ids: List of expense group ids @@ -176,10 +213,18 @@ def schedule_credit_card_purchase_creation(workspace_id: int, expense_group_ids: Q(tasklog__id__isnull=True) | ~Q(tasklog__status__in=['IN_PROGRESS', 'COMPLETE']), workspace_id=workspace_id, id__in=expense_group_ids, creditcardpurchase__id__isnull=True, exported_at__isnull=True ).all() + errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, expense_group_id__in=expense_group_ids).all() + chain_tasks = [] in_progress_expenses = [] for index, expense_group in enumerate(expense_groups): + error = errors.filter(workspace_id=workspace_id, expense_group=expense_group, is_resolved=False).first() + skip_export = validate_failing_export(is_auto_export, interval_hours, error) + if skip_export: + logger.info('Skipping expense group %s as it has %s errors', expense_group.id, error.repetition_count) + continue + task_log, _ = TaskLog.objects.get_or_create(workspace_id=expense_group.workspace_id, expense_group=expense_group, defaults={'status': 'ENQUEUED', 'type': 'CREATING_CREDIT_CARD_PURCHASE'}) if task_log.status not in ['IN_PROGRESS', 'ENQUEUED']: task_log.type = 'CREATING_CREDIT_CARD_PURCHASE' @@ -206,7 +251,7 @@ def schedule_credit_card_purchase_creation(workspace_id: int, expense_group_ids: __create_chain_and_run(fyle_credentials, in_progress_expenses, workspace_id, chain_tasks, fund_source) -def schedule_qbo_expense_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str): +def schedule_qbo_expense_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str, interval_hours: int): """ Schedule QBO expense creation :param expense_group_ids: List of expense group ids @@ -217,11 +262,18 @@ def schedule_qbo_expense_creation(workspace_id: int, expense_group_ids: List[str """ if expense_group_ids: expense_groups = ExpenseGroup.objects.filter(Q(tasklog__id__isnull=True) | ~Q(tasklog__status__in=['IN_PROGRESS', 'COMPLETE']), workspace_id=workspace_id, id__in=expense_group_ids, qboexpense__id__isnull=True, exported_at__isnull=True).all() + errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, expense_group_id__in=expense_group_ids).all() chain_tasks = [] in_progress_expenses = [] for index, expense_group in enumerate(expense_groups): + error = errors.filter(workspace_id=workspace_id, expense_group=expense_group, is_resolved=False).first() + skip_export = validate_failing_export(is_auto_export, interval_hours, error) + if skip_export: + logger.info('Skipping expense group %s as it has %s errors', expense_group.id, error.repetition_count) + continue + task_log, _ = TaskLog.objects.get_or_create( workspace_id=expense_group.workspace_id, expense_group=expense_group, defaults={'status': 'ENQUEUED', 'type': 'CREATING_EXPENSE' if expense_group.fund_source == 'PERSONAL' else 'CREATING_DEBIT_CARD_EXPENSE'} ) diff --git a/apps/workspaces/actions.py b/apps/workspaces/actions.py index 98389e51..72b31c4c 100644 --- a/apps/workspaces/actions.py +++ b/apps/workspaces/actions.py @@ -286,7 +286,8 @@ def export_to_qbo(workspace_id, export_mode=None, expense_group_ids=[], is_direc workspace_id=workspace_id, expense_group_ids=expense_group_ids, is_auto_export=export_mode == 'AUTO', - fund_source='PERSONAL' + fund_source='PERSONAL', + interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0 ) elif general_settings.reimbursable_expenses_object == 'EXPENSE': @@ -294,7 +295,8 @@ def export_to_qbo(workspace_id, export_mode=None, expense_group_ids=[], is_direc workspace_id=workspace_id, expense_group_ids=expense_group_ids, is_auto_export=export_mode == 'AUTO', - fund_source='PERSONAL' + fund_source='PERSONAL', + interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0 ) elif general_settings.reimbursable_expenses_object == 'CHECK': @@ -302,7 +304,8 @@ def export_to_qbo(workspace_id, export_mode=None, expense_group_ids=[], is_direc workspace_id=workspace_id, expense_group_ids=expense_group_ids, is_auto_export=export_mode == 'AUTO', - fund_source='PERSONAL' + fund_source='PERSONAL', + interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0 ) elif general_settings.reimbursable_expenses_object == 'JOURNAL ENTRY': @@ -310,7 +313,8 @@ def export_to_qbo(workspace_id, export_mode=None, expense_group_ids=[], is_direc workspace_id=workspace_id, expense_group_ids=expense_group_ids, is_auto_export=export_mode == 'AUTO', - fund_source='PERSONAL' + fund_source='PERSONAL', + interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0 ) if general_settings.corporate_credit_card_expenses_object: @@ -324,7 +328,8 @@ def export_to_qbo(workspace_id, export_mode=None, expense_group_ids=[], is_direc workspace_id=workspace_id, expense_group_ids=expense_group_ids, is_auto_export=export_mode == 'AUTO', - fund_source='CCC' + fund_source='CCC', + interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0 ) elif general_settings.corporate_credit_card_expenses_object == 'CREDIT CARD PURCHASE': @@ -332,7 +337,8 @@ def export_to_qbo(workspace_id, export_mode=None, expense_group_ids=[], is_direc workspace_id=workspace_id, expense_group_ids=expense_group_ids, is_auto_export=export_mode == 'AUTO', - fund_source='CCC' + fund_source='CCC', + interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0 ) elif general_settings.corporate_credit_card_expenses_object == 'DEBIT CARD EXPENSE': @@ -340,7 +346,8 @@ def export_to_qbo(workspace_id, export_mode=None, expense_group_ids=[], is_direc workspace_id=workspace_id, expense_group_ids=expense_group_ids, is_auto_export=export_mode == 'AUTO', - fund_source='CCC' + fund_source='CCC', + interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0 ) elif general_settings.corporate_credit_card_expenses_object == 'BILL': @@ -348,7 +355,8 @@ def export_to_qbo(workspace_id, export_mode=None, expense_group_ids=[], is_direc workspace_id=workspace_id, expense_group_ids=expense_group_ids, is_auto_export=export_mode == 'AUTO', - fund_source='CCC' + fund_source='CCC', + interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0 ) if is_expenses_exported: last_export_detail.last_exported_at = last_exported_at diff --git a/tests/test_quickbooks_online/test_tasks.py b/tests/test_quickbooks_online/test_tasks.py index e47e3c25..b86a74b3 100644 --- a/tests/test_quickbooks_online/test_tasks.py +++ b/tests/test_quickbooks_online/test_tasks.py @@ -1,10 +1,12 @@ import json import logging +from datetime import datetime import random from unittest import mock from django_q.models import Schedule from fyle_accounting_mappings.models import DestinationAttribute, EmployeeMapping, ExpenseAttribute, Mapping +from apps.quickbooks_online.models import CreditCardPurchaseLineitem from qbosdk.exceptions import WrongParamsError from apps.fyle.models import Expense, ExpenseGroup, Reimbursement @@ -1026,7 +1028,7 @@ def test_schedule_credit_card_purchase_creation(db): task_log.status = 'READY' task_log.save() - schedule_credit_card_purchase_creation(workspace_id, [17], False, 'CCC') + schedule_credit_card_purchase_creation(workspace_id, [17], False, 'CCC', 1) task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() assert task_log.type == 'CREATING_CREDIT_CARD_PURCHASE' @@ -1043,7 +1045,7 @@ def test_schedule_bills_creation(db): task_log.status = 'READY' task_log.save() - schedule_bills_creation(workspace_id, [23], False, 'PERSONAL') + schedule_bills_creation(workspace_id, [23], False, 'PERSONAL', 1) task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() assert task_log.type == 'CREATING_BILL' @@ -1060,7 +1062,7 @@ def test_schedule_cheques_creation(db): task_log.status = 'READY' task_log.save() - schedule_cheques_creation(workspace_id, [23], False, 'PERSONAL') + schedule_cheques_creation(workspace_id, [23], False, 'PERSONAL', 1) task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() assert task_log.type == 'CREATING_CHECK' @@ -1077,7 +1079,7 @@ def test_schedule_qbo_expense_creation(db): task_log.status = 'READY' task_log.save() - schedule_qbo_expense_creation(workspace_id, [23], False, 'PERSONAL') + schedule_qbo_expense_creation(workspace_id, [23], False, 'PERSONAL', 1) task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() assert task_log.type == 'CREATING_EXPENSE' @@ -1094,7 +1096,191 @@ def test_schedule_journal_entry_creation(db): task_log.status = 'READY' task_log.save() - schedule_journal_entry_creation(workspace_id, [23], False, 'PERSONAL') + schedule_journal_entry_creation(workspace_id, [23], False, 'PERSONAL', 1) task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() assert task_log.type == 'CREATING_JOURNAL_ENTRY' + + +def test_skipping_bill_creation(db, mocker): + workspace_id = 4 + + expense_group = ExpenseGroup.objects.get(id=23) + expense_group.exported_at = None + expense_group.save() + + error = Error.objects.filter(workspace_id=workspace_id, expense_group=expense_group).delete() + + error = Error.objects.create( + workspace_id=workspace_id, + type='NETSUITE_ERROR', + error_title='NetSuite System Error', + error_detail='An error occured in a upsert request: Please enter value(s) for: Location', + expense_group=expense_group, + repetition_count=106 + ) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + task_log.type = 'FETCHING_EXPENSES' + task_log.status = 'READY' + task_log.save() + + schedule_bills_creation(workspace_id, [23], True, 'PERSONAL', 1) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + assert task_log.type == 'FETCHING_EXPENSES' + + Error.objects.filter(id=error.id).update(updated_at=datetime(2024, 8, 20)) + + schedule_bills_creation(workspace_id, [23], True, 'CCC', 1) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + assert task_log.type == 'CREATING_BILL' + + +def test_skipping_journal_creation(db, mocker): + workspace_id = 4 + + expense_group = ExpenseGroup.objects.get(id=23) + expense_group.exported_at = None + expense_group.save() + + error = Error.objects.filter(workspace_id=workspace_id, expense_group=expense_group).delete() + + error = Error.objects.create( + workspace_id=workspace_id, + type='NETSUITE_ERROR', + error_title='NetSuite System Error', + error_detail='An error occured in a upsert request: Please enter value(s) for: Location', + expense_group=expense_group, + repetition_count=106 + ) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + task_log.type = 'FETCHING_EXPENSES' + task_log.status = 'READY' + task_log.save() + + schedule_journal_entry_creation(workspace_id, [23], True, 'CCC', 1) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + assert task_log.type == 'FETCHING_EXPENSES' + + Error.objects.filter(id=error.id).update(updated_at=datetime(2024, 8, 20)) + + schedule_journal_entry_creation(workspace_id, [23], True, 'PERSONAL', 1) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + assert task_log.type == 'CREATING_JOURNAL_ENTRY' + + +def test_skipping_qbo_expense_creation(db, mocker): + workspace_id = 4 + + expense_group = ExpenseGroup.objects.get(id=23) + expense_group.exported_at = None + expense_group.save() + + error = Error.objects.filter(workspace_id=workspace_id, expense_group=expense_group).delete() + + error = Error.objects.create( + workspace_id=workspace_id, + type='NETSUITE_ERROR', + error_title='NetSuite System Error', + error_detail='An error occured in a upsert request: Please enter value(s) for: Location', + expense_group=expense_group, + repetition_count=106 + ) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + task_log.type = 'FETCHING_EXPENSES' + task_log.status = 'READY' + task_log.save() + + schedule_qbo_expense_creation(workspace_id, [23], True, 'CCC', 1) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + assert task_log.type == 'FETCHING_EXPENSES' + + Error.objects.filter(id=error.id).update(updated_at=datetime(2024, 8, 20)) + + schedule_qbo_expense_creation(workspace_id, [23], True, 'PERSONAL', 1) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + assert task_log.type == 'CREATING_EXPENSE' + + +def test_skipping_credit_card_charge_creation(db, mocker): + workspace_id = 3 + + expense_group = ExpenseGroup.objects.get(id=17) + expense_group.exported_at = None + expense_group.save() + + credit = CreditCardPurchase.objects.get(expense_group_id=17) + CreditCardPurchaseLineitem.objects.filter(credit_card_purchase=credit).delete() + TaskLog.objects.filter(credit_card_purchase=credit).update(credit_card_purchase=None) + + credit.delete() + + error = Error.objects.create( + workspace_id=workspace_id, + type='NETSUITE_ERROR', + error_title='NetSuite System Error', + error_detail='An error occured in a upsert request: Please enter value(s) for: Location', + expense_group=expense_group, + repetition_count=106 + ) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + task_log.type = 'FETCHING_EXPENSES' + task_log.status = 'READY' + task_log.save() + + schedule_credit_card_purchase_creation(workspace_id, [17], True, 'CCC', 1) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + assert task_log.type == 'FETCHING_EXPENSES' + + Error.objects.filter(id=error.id).update(updated_at=datetime(2024, 8, 20)) + + schedule_credit_card_purchase_creation(workspace_id, [17], True, 'CCC', 1) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + assert task_log.type == 'CREATING_CREDIT_CARD_PURCHASE' + + +def test_skipping_cheque_creation(db, mocker): + workspace_id = 4 + + expense_group = ExpenseGroup.objects.get(id=23) + expense_group.exported_at = None + expense_group.save() + + error = Error.objects.filter(workspace_id=workspace_id, expense_group=expense_group).delete() + + error = Error.objects.create( + workspace_id=workspace_id, + type='NETSUITE_ERROR', + error_title='NetSuite System Error', + error_detail='An error occured in a upsert request: Please enter value(s) for: Location', + expense_group_id=expense_group.id, + repetition_count=106 + ) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + task_log.type = 'FETCHING_EXPENSES' + task_log.status = 'READY' + task_log.save() + + schedule_cheques_creation(workspace_id, [23], True, 'PERSONAL', 1) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + assert task_log.type == 'FETCHING_EXPENSES' + + Error.objects.filter(id=error.id).update(updated_at=datetime(2024, 8, 20)) + + schedule_cheques_creation(workspace_id, [23], True, 'CCC', 1) + + task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first() + assert task_log.type == 'CREATING_CHECK'