Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Max retry for exports #662

Merged
merged 5 commits into from
Aug 29, 2024
Merged

Max retry for exports #662

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e13add6
Max retry for exports
Ashutosh619-sudo Aug 21, 2024
b0ac814
proper tests
Ashutosh619-sudo Aug 26, 2024
4543e76
remove loggers
Ashutosh619-sudo Aug 26, 2024
71f7129
remove loggers
Ashutosh619-sudo Aug 26, 2024
f8dc902
flake resolved
Ashutosh619-sudo Aug 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 60 additions & 7 deletions apps/quickbooks_online/queue.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,36 @@
from datetime import datetime, timedelta
import logging
from datetime import datetime, timedelta, timezone
from typing import List

from django.db.models import Q
from django_q.models import Schedule
from django_q.tasks import Chain, async_task

from apps.fyle.models import Expense, ExpenseGroup
from apps.tasks.models import TaskLog
from apps.tasks.models import TaskLog, Error
from apps.workspaces.models import FyleCredential, WorkspaceGeneralSettings


logger = logging.getLogger(__name__)
logger.level = logging.INFO


def async_run_post_configration_triggers(workspace_general_settings: WorkspaceGeneralSettings):
async_task('apps.quickbooks_online.tasks.async_sync_accounts', int(workspace_general_settings.workspace_id), q_options={'cluster': 'import'})


def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str):
def validate_failing_export(is_auto_export: bool, interval_hours: int, error: Error):
"""
Validate failing export
:param is_auto_export: Is auto export
:param interval_hours: Interval hours
:param error: Error
"""
# If auto export is enabled and interval hours is set and error repetition count is greater than 100, export only once a day
return is_auto_export and interval_hours and error and error.repetition_count > 100 and datetime.now().replace(tzinfo=timezone.utc) - error.updated_at <= timedelta(hours=24)

Comment on lines +22 to +30
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Reminder: Add unit tests for validate_failing_export.

The function is correctly implemented but lacks unit tests. Ensure to add unit tests to validate its behavior.

Do you want me to generate the unit testing code or open a GitHub issue to track this task?


def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str, interval_hours: int):
"""
Schedule bills creation
:param expense_group_ids: List of expense group ids
Expand All @@ -26,10 +42,17 @@ def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str], is_
if expense_group_ids:
expense_groups = ExpenseGroup.objects.filter(Q(tasklog__id__isnull=True) | ~Q(tasklog__status__in=['IN_PROGRESS', 'COMPLETE']), workspace_id=workspace_id, id__in=expense_group_ids, bill__id__isnull=True, exported_at__isnull=True).all()

errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, expense_group_id__in=expense_group_ids).all()

chain_tasks = []
in_progress_expenses = []

for index, expense_group in enumerate(expense_groups):
error = errors.filter(workspace_id=workspace_id, expense_group=expense_group, is_resolved=False).first()
skip_export = validate_failing_export(is_auto_export, interval_hours, error)
if skip_export:
logger.info('Skipping expense group %s as it has %s errors', expense_group.id, error.repetition_count)
continue
task_log, _ = TaskLog.objects.get_or_create(workspace_id=expense_group.workspace_id, expense_group=expense_group, defaults={'status': 'ENQUEUED', 'type': 'CREATING_BILL'})
if task_log.status not in ['IN_PROGRESS', 'ENQUEUED']:
task_log.type = 'CREATING_BILL'
Expand Down Expand Up @@ -79,7 +102,7 @@ def __create_chain_and_run(fyle_credentials: FyleCredential, in_progress_expense
chain.run()


def schedule_cheques_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str):
def schedule_cheques_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str, interval_hours: int):
"""
Schedule cheque creation
:param expense_group_ids: List of expense group ids
Expand All @@ -91,10 +114,17 @@ def schedule_cheques_creation(workspace_id: int, expense_group_ids: List[str], i
if expense_group_ids:
expense_groups = ExpenseGroup.objects.filter(Q(tasklog__id__isnull=True) | ~Q(tasklog__status__in=['IN_PROGRESS', 'COMPLETE']), workspace_id=workspace_id, id__in=expense_group_ids, cheque__id__isnull=True, exported_at__isnull=True).all()

errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, expense_group_id__in=expense_group_ids).all()

chain_tasks = []
in_progress_expenses = []

for index, expense_group in enumerate(expense_groups):
error = errors.filter(workspace_id=workspace_id, expense_group=expense_group, is_resolved=False).first()
skip_export = validate_failing_export(is_auto_export, interval_hours, error)
if skip_export:
logger.info('Skipping expense group %s as it has %s errors', expense_group.id, error.repetition_count)
continue
task_log, _ = TaskLog.objects.get_or_create(workspace_id=expense_group.workspace_id, expense_group=expense_group, defaults={'status': 'ENQUEUED', 'type': 'CREATING_CHECK'})
if task_log.status not in ['IN_PROGRESS', 'ENQUEUED']:
task_log.type = 'CREATING_CHECK'
Expand All @@ -121,7 +151,7 @@ def schedule_cheques_creation(workspace_id: int, expense_group_ids: List[str], i
__create_chain_and_run(fyle_credentials, in_progress_expenses, workspace_id, chain_tasks, fund_source)


def schedule_journal_entry_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str):
def schedule_journal_entry_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str, interval_hours: int):
"""
Schedule journal_entry creation
:param expense_group_ids: List of expense group ids
Expand All @@ -133,10 +163,17 @@ def schedule_journal_entry_creation(workspace_id: int, expense_group_ids: List[s
Q(tasklog__id__isnull=True) | ~Q(tasklog__status__in=['IN_PROGRESS', 'COMPLETE']), workspace_id=workspace_id, id__in=expense_group_ids, journalentry__id__isnull=True, exported_at__isnull=True
).all()

errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, expense_group_id__in=expense_group_ids).all()

chain_tasks = []
in_progress_expenses = []

for index, expense_group in enumerate(expense_groups):
error = errors.filter(workspace_id=workspace_id, expense_group=expense_group, is_resolved=False).first()
skip_export = validate_failing_export(is_auto_export, interval_hours, error)
if skip_export:
logger.info('Skipping expense group %s as it has %s errors', expense_group.id, error.repetition_count)
continue
task_log, _ = TaskLog.objects.get_or_create(workspace_id=expense_group.workspace_id, expense_group=expense_group, defaults={'status': 'ENQUEUED', 'type': 'CREATING_JOURNAL_ENTRY'})
if task_log.status not in ['IN_PROGRESS', 'ENQUEUED']:
task_log.type = 'CREATING_JOURNAL_ENTRY'
Expand All @@ -162,7 +199,7 @@ def schedule_journal_entry_creation(workspace_id: int, expense_group_ids: List[s
__create_chain_and_run(fyle_credentials, in_progress_expenses, workspace_id, chain_tasks, fund_source)


def schedule_credit_card_purchase_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str):
def schedule_credit_card_purchase_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str, interval_hours: int):
"""
Schedule credit card purchase creation
:param expense_group_ids: List of expense group ids
Expand All @@ -176,10 +213,18 @@ def schedule_credit_card_purchase_creation(workspace_id: int, expense_group_ids:
Q(tasklog__id__isnull=True) | ~Q(tasklog__status__in=['IN_PROGRESS', 'COMPLETE']), workspace_id=workspace_id, id__in=expense_group_ids, creditcardpurchase__id__isnull=True, exported_at__isnull=True
).all()

errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, expense_group_id__in=expense_group_ids).all()

chain_tasks = []
in_progress_expenses = []

for index, expense_group in enumerate(expense_groups):
error = errors.filter(workspace_id=workspace_id, expense_group=expense_group, is_resolved=False).first()
skip_export = validate_failing_export(is_auto_export, interval_hours, error)
if skip_export:
logger.info('Skipping expense group %s as it has %s errors', expense_group.id, error.repetition_count)
continue

task_log, _ = TaskLog.objects.get_or_create(workspace_id=expense_group.workspace_id, expense_group=expense_group, defaults={'status': 'ENQUEUED', 'type': 'CREATING_CREDIT_CARD_PURCHASE'})
if task_log.status not in ['IN_PROGRESS', 'ENQUEUED']:
task_log.type = 'CREATING_CREDIT_CARD_PURCHASE'
Expand All @@ -206,7 +251,7 @@ def schedule_credit_card_purchase_creation(workspace_id: int, expense_group_ids:
__create_chain_and_run(fyle_credentials, in_progress_expenses, workspace_id, chain_tasks, fund_source)


def schedule_qbo_expense_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str):
def schedule_qbo_expense_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str, interval_hours: int):
"""
Schedule QBO expense creation
:param expense_group_ids: List of expense group ids
Expand All @@ -218,10 +263,18 @@ def schedule_qbo_expense_creation(workspace_id: int, expense_group_ids: List[str
if expense_group_ids:
expense_groups = ExpenseGroup.objects.filter(Q(tasklog__id__isnull=True) | ~Q(tasklog__status__in=['IN_PROGRESS', 'COMPLETE']), workspace_id=workspace_id, id__in=expense_group_ids, qboexpense__id__isnull=True, exported_at__isnull=True).all()

errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, expense_group_id__in=expense_group_ids).all()

chain_tasks = []
in_progress_expenses = []

for index, expense_group in enumerate(expense_groups):
error = errors.filter(workspace_id=workspace_id, expense_group=expense_group, is_resolved=False).first()
skip_export = validate_failing_export(is_auto_export, interval_hours, error)
if skip_export:
logger.info('Skipping expense group %s as it has %s errors', expense_group.id, error.repetition_count)
continue

task_log, _ = TaskLog.objects.get_or_create(
workspace_id=expense_group.workspace_id, expense_group=expense_group, defaults={'status': 'ENQUEUED', 'type': 'CREATING_EXPENSE' if expense_group.fund_source == 'PERSONAL' else 'CREATING_DEBIT_CARD_EXPENSE'}
)
Expand Down
24 changes: 16 additions & 8 deletions apps/workspaces/actions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,31 +286,35 @@ def export_to_qbo(workspace_id, export_mode=None, expense_group_ids=[], is_direc
workspace_id=workspace_id,
expense_group_ids=expense_group_ids,
is_auto_export=export_mode == 'AUTO',
fund_source='PERSONAL'
fund_source='PERSONAL',
interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0
)

elif general_settings.reimbursable_expenses_object == 'EXPENSE':
schedule_qbo_expense_creation(
workspace_id=workspace_id,
expense_group_ids=expense_group_ids,
is_auto_export=export_mode == 'AUTO',
fund_source='PERSONAL'
fund_source='PERSONAL',
interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0
)

elif general_settings.reimbursable_expenses_object == 'CHECK':
schedule_cheques_creation(
workspace_id=workspace_id,
expense_group_ids=expense_group_ids,
is_auto_export=export_mode == 'AUTO',
fund_source='PERSONAL'
fund_source='PERSONAL',
interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0
)

elif general_settings.reimbursable_expenses_object == 'JOURNAL ENTRY':
schedule_journal_entry_creation(
workspace_id=workspace_id,
expense_group_ids=expense_group_ids,
is_auto_export=export_mode == 'AUTO',
fund_source='PERSONAL'
fund_source='PERSONAL',
interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0
)

if general_settings.corporate_credit_card_expenses_object:
Expand All @@ -324,31 +328,35 @@ def export_to_qbo(workspace_id, export_mode=None, expense_group_ids=[], is_direc
workspace_id=workspace_id,
expense_group_ids=expense_group_ids,
is_auto_export=export_mode == 'AUTO',
fund_source='CCC'
fund_source='CCC',
interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0
)

elif general_settings.corporate_credit_card_expenses_object == 'CREDIT CARD PURCHASE':
schedule_credit_card_purchase_creation(
workspace_id=workspace_id,
expense_group_ids=expense_group_ids,
is_auto_export=export_mode == 'AUTO',
fund_source='CCC'
fund_source='CCC',
interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0
)

elif general_settings.corporate_credit_card_expenses_object == 'DEBIT CARD EXPENSE':
schedule_qbo_expense_creation(
workspace_id=workspace_id,
expense_group_ids=expense_group_ids,
is_auto_export=export_mode == 'AUTO',
fund_source='CCC'
fund_source='CCC',
interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0
)

elif general_settings.corporate_credit_card_expenses_object == 'BILL':
schedule_bills_creation(
workspace_id=workspace_id,
expense_group_ids=expense_group_ids,
is_auto_export=export_mode == 'AUTO',
fund_source='CCC'
fund_source='CCC',
interval_hours=workspace_schedule.interval_hours if workspace_schedule else 0
)
if is_expenses_exported:
last_export_detail.last_exported_at = last_exported_at
Expand Down
10 changes: 5 additions & 5 deletions tests/test_quickbooks_online/test_tasks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1026,7 +1026,7 @@ def test_schedule_credit_card_purchase_creation(db):
task_log.status = 'READY'
task_log.save()

schedule_credit_card_purchase_creation(workspace_id, [17], False, 'CCC')
schedule_credit_card_purchase_creation(workspace_id, [17], False, 'CCC', 1)

task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first()
assert task_log.type == 'CREATING_CREDIT_CARD_PURCHASE'
Expand All @@ -1043,7 +1043,7 @@ def test_schedule_bills_creation(db):
task_log.status = 'READY'
task_log.save()

schedule_bills_creation(workspace_id, [23], False, 'PERSONAL')
schedule_bills_creation(workspace_id, [23], False, 'PERSONAL', 1)

task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first()
assert task_log.type == 'CREATING_BILL'
Expand All @@ -1060,7 +1060,7 @@ def test_schedule_cheques_creation(db):
task_log.status = 'READY'
task_log.save()

schedule_cheques_creation(workspace_id, [23], False, 'PERSONAL')
schedule_cheques_creation(workspace_id, [23], False, 'PERSONAL', 1)

task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first()
assert task_log.type == 'CREATING_CHECK'
Expand All @@ -1077,7 +1077,7 @@ def test_schedule_qbo_expense_creation(db):
task_log.status = 'READY'
task_log.save()

schedule_qbo_expense_creation(workspace_id, [23], False, 'PERSONAL')
schedule_qbo_expense_creation(workspace_id, [23], False, 'PERSONAL', 1)

task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first()
assert task_log.type == 'CREATING_EXPENSE'
Expand All @@ -1094,7 +1094,7 @@ def test_schedule_journal_entry_creation(db):
task_log.status = 'READY'
task_log.save()

schedule_journal_entry_creation(workspace_id, [23], False, 'PERSONAL')
schedule_journal_entry_creation(workspace_id, [23], False, 'PERSONAL', 1)

task_log = TaskLog.objects.filter(expense_group_id=expense_group.id).first()
assert task_log.type == 'CREATING_JOURNAL_ENTRY'
Loading