From 2fe4a3ae87f079beb4acd9ed2d3276c9d13f56ff Mon Sep 17 00:00:00 2001 From: Ashutosh singh <55102089+Ashutosh619-sudo@users.noreply.github.com> Date: Thu, 29 Aug 2024 14:55:08 +0530 Subject: [PATCH] Max retry for exports limit (#161) * Max retry for exports limit * test case resolved --- apps/business_central/exports/helpers.py | 12 ++++++++++++ .../exports/journal_entry/queues.py | 18 ++++++++++++++++-- .../exports/journal_entry/tasks.py | 4 ++-- .../exports/purchase_invoice/queues.py | 19 +++++++++++++++---- .../exports/purchase_invoice/tasks.py | 4 ++-- apps/workspaces/tasks.py | 14 ++++++++++---- tests/test_business_central/test_queues.py | 16 ++++++++++++---- tests/test_business_central/test_tasks.py | 4 ++-- 8 files changed, 71 insertions(+), 20 deletions(-) diff --git a/apps/business_central/exports/helpers.py b/apps/business_central/exports/helpers.py index 86f4056..2a84464 100644 --- a/apps/business_central/exports/helpers.py +++ b/apps/business_central/exports/helpers.py @@ -1,6 +1,7 @@ import itertools import logging import traceback +from datetime import datetime, timezone, timedelta from fyle_accounting_mappings.models import CategoryMapping, EmployeeMapping, ExpenseAttribute, Mapping from fyle_integrations_platform_connector import PlatformConnector @@ -205,3 +206,14 @@ def load_attachments( accounting_export.workspace_id, {"error": error}, ) + + +def validate_failing_export(is_auto_export: bool, interval_hours: int, error: Error): + """ + Validate failing export + :param is_auto_export: Is auto export + :param interval_hours: Interval hours + :param error: Error + """ + # If auto export is enabled and interval hours is set and error repetition count is greater than 100, export only once a day + return is_auto_export and interval_hours and error and error.repetition_count > 100 and datetime.now().replace(tzinfo=timezone.utc) - error.updated_at <= timedelta(hours=24) diff --git a/apps/business_central/exports/journal_entry/queues.py b/apps/business_central/exports/journal_entry/queues.py index 94405ca..3e941d7 100644 --- a/apps/business_central/exports/journal_entry/queues.py +++ b/apps/business_central/exports/journal_entry/queues.py @@ -1,13 +1,20 @@ from typing import List +import logging + from django.db.models import Q from django_q.tasks import Chain -from apps.accounting_exports.models import AccountingExport +from apps.accounting_exports.models import AccountingExport, Error +from apps.business_central.exports.helpers import validate_failing_export from apps.workspaces.models import FyleCredential -def check_accounting_export_and_start_import(workspace_id: int, accounting_export_ids: List[str]): +logger = logging.getLogger(__name__) +logger.level = logging.INFO + + +def check_accounting_export_and_start_import(workspace_id: int, accounting_export_ids: List[str], is_auto_export: bool, interval_hours: int): """ Check accounting export group and start export """ @@ -18,10 +25,17 @@ def check_accounting_export_and_start_import(workspace_id: int, accounting_expor workspace_id=workspace_id, id__in=accounting_export_ids, journal_entry__id__isnull=True, exported_at__isnull=True).all() + errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, accounting_export_id__in=accounting_export_ids).all() + chain = Chain() chain.append('apps.fyle.helpers.sync_dimensions', fyle_credentials) for index, accounting_export_group in enumerate(accounting_exports): + error = errors.filter(workspace_id=workspace_id, accounting_export=accounting_export_group, is_resolved=False).first() + skip_export = validate_failing_export(is_auto_export, interval_hours, error) + if skip_export: + logger.info('Skipping expense group %s as it has %s errors', accounting_export_group.id, error.repetition_count) + continue accounting_export, _ = AccountingExport.objects.update_or_create( workspace_id=accounting_export_group.workspace_id, id=accounting_export_group.id, diff --git a/apps/business_central/exports/journal_entry/tasks.py b/apps/business_central/exports/journal_entry/tasks.py index 37677ac..243ecc6 100644 --- a/apps/business_central/exports/journal_entry/tasks.py +++ b/apps/business_central/exports/journal_entry/tasks.py @@ -26,11 +26,11 @@ def __init__(self): self.body_model = JournalEntry self.lineitem_model = JournalEntryLineItems - def trigger_export(self, workspace_id, accounting_export_ids): + def trigger_export(self, workspace_id, accounting_export_ids, is_auto_export, interval_hours): ''' Trigger the import process for the Project module. ''' - check_accounting_export_and_start_import(workspace_id, accounting_export_ids) + check_accounting_export_and_start_import(workspace_id=workspace_id, accounting_export_ids=accounting_export_ids, is_auto_export=is_auto_export, interval_hours=interval_hours) def __construct_journal_entry(self, body: JournalEntry, lineitems: List[JournalEntryLineItems]) -> Dict: ''' diff --git a/apps/business_central/exports/purchase_invoice/queues.py b/apps/business_central/exports/purchase_invoice/queues.py index cd0a408..48aa64c 100644 --- a/apps/business_central/exports/purchase_invoice/queues.py +++ b/apps/business_central/exports/purchase_invoice/queues.py @@ -1,13 +1,18 @@ from typing import List +import logging from django.db.models import Q from django_q.tasks import Chain -from apps.accounting_exports.models import AccountingExport +from apps.accounting_exports.models import AccountingExport, Error +from apps.business_central.exports.helpers import validate_failing_export from apps.workspaces.models import FyleCredential +logger = logging.getLogger(__name__) +logger.level = logging.INFO -def check_accounting_export_and_start_import(workspace_id: int, accounting_export_ids: List[str]): + +def check_accounting_export_and_start_import(workspace_id: int, accounting_export_ids: List[str], is_auto_export: bool, interval_hours: int): """ Check accounting export group and start export """ @@ -15,13 +20,19 @@ def check_accounting_export_and_start_import(workspace_id: int, accounting_expor fyle_credentials = FyleCredential.objects.filter(workspace_id=workspace_id).first() accounting_exports = AccountingExport.objects.filter(~Q(status__in=['IN_PROGRESS', 'COMPLETE', 'EXPORT_QUEUED']), - workspace_id=workspace_id, id__in=accounting_export_ids, purchase_invoice__id__isnull=True, - exported_at__isnull=True).all() + workspace_id=workspace_id, id__in=accounting_export_ids, purchase_invoice__id__isnull=True, exported_at__isnull=True).all() + + errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, accounting_export_id__in=accounting_export_ids).all() chain = Chain() chain.append('apps.fyle.helpers.sync_dimensions', fyle_credentials) for index, accounting_export_group in enumerate(accounting_exports): + error = errors.filter(workspace_id=workspace_id, accounting_export=accounting_export_group, is_resolved=False).first() + skip_export = validate_failing_export(is_auto_export, interval_hours, error) + if skip_export: + logger.info('Skipping expense group %s as it has %s errors', accounting_export_group.id, error.repetition_count) + continue accounting_export, _ = AccountingExport.objects.update_or_create( workspace_id=accounting_export_group.workspace_id, id=accounting_export_group.id, diff --git a/apps/business_central/exports/purchase_invoice/tasks.py b/apps/business_central/exports/purchase_invoice/tasks.py index eb1e401..896576b 100644 --- a/apps/business_central/exports/purchase_invoice/tasks.py +++ b/apps/business_central/exports/purchase_invoice/tasks.py @@ -27,11 +27,11 @@ def __init__(self): self.body_model = PurchaseInvoice self.lineitem_model = PurchaseInvoiceLineitems - def trigger_export(self, workspace_id, accounting_export_ids): + def trigger_export(self, workspace_id, accounting_export_ids, is_auto_export, interval_hours): ''' Trigger the import process for the Project module. ''' - check_accounting_export_and_start_import(workspace_id, accounting_export_ids) + check_accounting_export_and_start_import(workspace_id=workspace_id, accounting_export_ids=accounting_export_ids, is_auto_export=is_auto_export, interval_hours=interval_hours) def __construct_purchase_invoice(self, body: PurchaseInvoice, lineitems: List[PurchaseInvoiceLineitems]) -> Dict: ''' diff --git a/apps/workspaces/tasks.py b/apps/workspaces/tasks.py index 098e095..934da22 100644 --- a/apps/workspaces/tasks.py +++ b/apps/workspaces/tasks.py @@ -37,6 +37,9 @@ def run_import_export(workspace_id: int, export_mode = None): workspace_id=workspace_id ) + interval_hours = advance_settings.interval_hours + is_auto_export = advance_settings.schedule_is_enabled + last_exported_at = datetime.now() is_expenses_exported = False @@ -61,7 +64,7 @@ def run_import_export(workspace_id: int, export_mode = None): is_expenses_exported = True export = export_map.get(export_settings.reimbursable_expenses_export_type, None) if export: - export.trigger_export(workspace_id=workspace_id, accounting_export_ids=accounting_export_ids) + export.trigger_export(workspace_id=workspace_id, accounting_export_ids=accounting_export_ids, is_auto_export=is_auto_export, interval_hours=interval_hours) # For Credit Card Expenses if export_settings.credit_card_expense_export_type: @@ -78,7 +81,7 @@ def run_import_export(workspace_id: int, export_mode = None): is_expenses_exported = True export = export_map.get(export_settings.credit_card_expense_export_type, None) if export: - export.trigger_export(workspace_id=workspace_id, accounting_export_ids=accounting_export_ids) + export.trigger_export(workspace_id=workspace_id, accounting_export_ids=accounting_export_ids, is_auto_export=is_auto_export, interval_hours=interval_hours) if is_expenses_exported: accounting_summary.last_exported_at = last_exported_at @@ -139,6 +142,9 @@ def export_to_business_central(workspace_id: int): export_settings = ExportSetting.objects.get(workspace_id=workspace_id) advance_settings = AdvancedSetting.objects.filter(workspace_id=workspace_id).first() + is_auto_export = False + interval_hours = 0 + # Update or create an AccountingExportSummary for the workspace accounting_summary, _ = AccountingExportSummary.objects.update_or_create(workspace_id=workspace_id) @@ -166,7 +172,7 @@ def export_to_business_central(workspace_id: int): # Get the appropriate export class and trigger the export export = export_map.get(export_settings.reimbursable_expenses_export_type, None) if export: - export.trigger_export(workspace_id=workspace_id, accounting_export_ids=accounting_export_ids) + export.trigger_export(workspace_id=workspace_id, accounting_export_ids=accounting_export_ids, is_auto_export=is_auto_export, interval_hours=interval_hours) # Check and export credit card expenses if configured if export_settings.credit_card_expense_export_type: @@ -180,7 +186,7 @@ def export_to_business_central(workspace_id: int): # Get the appropriate export class and trigger the export export = export_map.get(export_settings.credit_card_expense_export_type, None) if export: - export.trigger_export(workspace_id=workspace_id, accounting_export_ids=accounting_export_ids) + export.trigger_export(workspace_id=workspace_id, accounting_export_ids=accounting_export_ids, is_auto_export=is_auto_export, interval_hours=interval_hours) # Update the accounting summary if expenses are exported if is_expenses_exported: diff --git a/tests/test_business_central/test_queues.py b/tests/test_business_central/test_queues.py index 61c6d59..3c134bf 100644 --- a/tests/test_business_central/test_queues.py +++ b/tests/test_business_central/test_queues.py @@ -29,7 +29,9 @@ def test_check_accounting_export_and_start_import_journal_entry( check_accounting_export_and_start_import_journal_entry( accounting_export.workspace_id, - [accounting_export.id] + [accounting_export.id], + False, + 0 ) accounting_export.refresh_from_db() @@ -42,7 +44,9 @@ def test_check_accounting_export_and_start_import_journal_entry( check_accounting_export_and_start_import_journal_entry( accounting_export.workspace_id, - [accounting_export.id] + [accounting_export.id], + False, + 0 ) accounting_export.refresh_from_db() @@ -70,7 +74,9 @@ def test_check_accounting_export_and_start_import_purchase_invoice( check_accounting_export_and_start_import_purchase_invoice( accounting_export.workspace_id, - [accounting_export.id] + [accounting_export.id], + False, + 0 ) accounting_export.refresh_from_db() @@ -83,7 +89,9 @@ def test_check_accounting_export_and_start_import_purchase_invoice( check_accounting_export_and_start_import_purchase_invoice( accounting_export.workspace_id, - [accounting_export.id] + [accounting_export.id], + False, + 0 ) accounting_export.refresh_from_db() diff --git a/tests/test_business_central/test_tasks.py b/tests/test_business_central/test_tasks.py index 45bf26a..6fdc75c 100644 --- a/tests/test_business_central/test_tasks.py +++ b/tests/test_business_central/test_tasks.py @@ -18,7 +18,7 @@ def test_trigger_export_journal_entry(db, mocker): ) export_journal_entry = ExportJournalEntry() - export_journal_entry.trigger_export(1, [1]) + export_journal_entry.trigger_export(1, [1], False, 0) assert True @@ -140,7 +140,7 @@ def test_trigger_export_purchase_invoice(db, mocker): ) export_purchase_invoice = ExportPurchaseInvoice() - export_purchase_invoice.trigger_export(1, [1]) + export_purchase_invoice.trigger_export(1, [1], False, 0) assert True