From 8836a97caff95ce83cb20c5ec04685d717cff5c6 Mon Sep 17 00:00:00 2001 From: ashwin1111 Date: Wed, 11 Oct 2023 20:19:40 +0530 Subject: [PATCH] Add support for receiving webhook event --- apps/fyle/helpers.py | 66 +++++++++++++++++++- apps/fyle/queue.py | 12 ++++ apps/fyle/tasks.py | 122 ++++++++++++++++++++++++------------- apps/fyle/urls.py | 2 + apps/fyle/views.py | 14 ++++- apps/workspaces/actions.py | 92 +++++++++++++++++++++++++++- apps/workspaces/tasks.py | 92 +--------------------------- apps/workspaces/views.py | 2 +- requirements.txt | 2 +- 9 files changed, 264 insertions(+), 140 deletions(-) diff --git a/apps/fyle/helpers.py b/apps/fyle/helpers.py index 0269088b..c04f5edf 100644 --- a/apps/fyle/helpers.py +++ b/apps/fyle/helpers.py @@ -1,11 +1,20 @@ import json +import traceback +import logging from typing import List, Union import requests from django.conf import settings from django.db.models import Q -from apps.fyle.models import ExpenseFilter +from apps.fyle.models import ExpenseFilter, ExpenseGroupSettings +from apps.tasks.models import TaskLog +from apps.workspaces.models import WorkspaceGeneralSettings + +logger = logging.getLogger(__name__) +logger.level = logging.INFO + +SOURCE_ACCOUNT_MAP = {'PERSONAL': 'PERSONAL_CASH_ACCOUNT', 'CCC': 'PERSONAL_CORPORATE_CREDIT_CARD_ACCOUNT'} def post_request(url, body, refresh_token=None): @@ -170,3 +179,58 @@ def get_updated_accounting_export_summary( 'url': url, 'synced': is_synced } + + +def get_source_account_type(fund_source: List[str]) -> List[str]: + """ + Get source account type + :param fund_source: fund source + :return: source account type + """ + source_account_type = [] + for source in fund_source: + source_account_type.append(SOURCE_ACCOUNT_MAP[source]) + + return source_account_type + + +def get_fund_source(workspace_id: int) -> List[str]: + """ + Get fund source + :param workspace_id: workspace id + :return: fund source + """ + general_settings = WorkspaceGeneralSettings.objects.get(workspace_id=workspace_id) + fund_source = [] + if general_settings.reimbursable_expenses_object: + fund_source.append('PERSONAL') + if general_settings.corporate_credit_card_expenses_object: + fund_source.append('CCC') + + return fund_source + + +def get_filter_credit_expenses(expense_group_settings: ExpenseGroupSettings) -> bool: + """ + Get filter credit expenses + :param expense_group_settings: expense group settings + :return: filter credit expenses + """ + filter_credit_expenses = True + if expense_group_settings.import_card_credits: + filter_credit_expenses = False + + return filter_credit_expenses + + +def handle_import_exception(task_log: TaskLog) -> None: + """ + Handle import exception + :param task_log: task log + :return: None + """ + error = traceback.format_exc() + task_log.detail = {'error': error} + task_log.status = 'FATAL' + task_log.save() + logger.error('Something unexpected happened workspace_id: %s %s', task_log.workspace_id, task_log.detail) diff --git a/apps/fyle/queue.py b/apps/fyle/queue.py index 0125dd95..68cec648 100644 --- a/apps/fyle/queue.py +++ b/apps/fyle/queue.py @@ -10,3 +10,15 @@ def async_post_accounting_export_summary(org_id: str, workspace_id: int) -> None """ # This function calls post_accounting_export_summary asynchrously async_task('apps.fyle.tasks.post_accounting_export_summary', org_id, workspace_id) + + +def async_import_and_export_expenses(body: dict) -> None: + """ + Async'ly import and export expenses + :param body: body + :return: None + """ + if body.get('action') == 'ACCOUNTING_EXPORT_INITIATED' and body.get('data'): + report_id = body['data']['id'] + org_id = body['data']['org_id'] + async_task('apps.fyle.tasks.import_and_export_expenses', report_id, org_id) diff --git a/apps/fyle/tasks.py b/apps/fyle/tasks.py index 242d2a76..b938edb8 100644 --- a/apps/fyle/tasks.py +++ b/apps/fyle/tasks.py @@ -1,39 +1,39 @@ import logging -import traceback from datetime import datetime -from typing import List +from typing import List, Dict from django.db import transaction from fyle.platform.exceptions import InvalidTokenError as FyleInvalidTokenError, RetryException from fyle_integrations_platform_connector import PlatformConnector -from apps.fyle.helpers import construct_expense_filter_query -from apps.fyle.models import Expense, ExpenseFilter, ExpenseGroup, ExpenseGroupSettings +from apps.fyle.models import Expense, ExpenseGroupSettings, ExpenseFilter, ExpenseGroup from apps.tasks.models import TaskLog -from apps.workspaces.models import FyleCredential, Workspace, WorkspaceGeneralSettings +from apps.workspaces.models import FyleCredential, Workspace +from apps.workspaces.actions import export_to_qbo from .actions import ( - mark_expenses_as_skipped, mark_accounting_export_summary_as_synced, - bulk_post_accounting_export_summary + bulk_post_accounting_export_summary, + mark_expenses_as_skipped +) +from .helpers import ( + handle_import_exception, + get_source_account_type, + get_fund_source, + get_filter_credit_expenses, + construct_expense_filter_query ) from .queue import async_post_accounting_export_summary + logger = logging.getLogger(__name__) logger.level = logging.INFO -SOURCE_ACCOUNT_MAP = {'PERSONAL': 'PERSONAL_CASH_ACCOUNT', 'CCC': 'PERSONAL_CORPORATE_CREDIT_CARD_ACCOUNT'} - def get_task_log_and_fund_source(workspace_id: int): task_log, _ = TaskLog.objects.update_or_create(workspace_id=workspace_id, type='FETCHING_EXPENSES', defaults={'status': 'IN_PROGRESS'}) - general_settings = WorkspaceGeneralSettings.objects.get(workspace_id=workspace_id) - fund_source = [] - if general_settings.reimbursable_expenses_object: - fund_source.append('PERSONAL') - if general_settings.corporate_credit_card_expenses_object: - fund_source.append('CCC') + fund_source = get_fund_source(workspace_id) return task_log, fund_source @@ -55,6 +55,32 @@ def create_expense_groups(workspace_id: int, fund_source: List[str], task_log: T return task_log +def group_expenses_and_save(expenses: List[Dict], task_log: TaskLog, workspace: Workspace): + expense_objects = Expense.create_expense_objects(expenses, workspace.id) + expense_filters = ExpenseFilter.objects.filter(workspace_id=workspace.id).order_by('rank') + filtered_expenses = expense_objects + if expense_filters: + expenses_object_ids = [expense_object.id for expense_object in expense_objects] + final_query = construct_expense_filter_query(expense_filters) + + mark_expenses_as_skipped(final_query, expenses_object_ids, workspace) + async_post_accounting_export_summary(workspace.fyle_org_id, workspace.id) + + filtered_expenses = Expense.objects.filter( + is_skipped=False, + id__in=expenses_object_ids, + expensegroup__isnull=True, + org_id=workspace.fyle_org_id + ) + + ExpenseGroup.create_expense_groups_by_report_id_fund_source( + filtered_expenses, workspace.id + ) + + task_log.status = 'COMPLETE' + task_log.save() + + def async_create_expense_groups(workspace_id: int, fund_source: List[str], task_log: TaskLog): try: with transaction.atomic(): @@ -72,20 +98,13 @@ def async_create_expense_groups(workspace_id: int, fund_source: List[str], task_ platform = PlatformConnector(fyle_credentials) - source_account_type = [] - for source in fund_source: - source_account_type.append(SOURCE_ACCOUNT_MAP[source]) - - filter_credit_expenses = True - if expense_group_settings.import_card_credits: - filter_credit_expenses = False + filter_credit_expenses = get_filter_credit_expenses(expense_group_settings) expenses = [] reimbursable_expense_count = 0 settled_at, approved_at, last_paid_at = None, None, None if 'PERSONAL' in fund_source: - if expense_group_settings.expense_state == 'PAYMENT_PROCESSING': settled_at = last_synced_at @@ -127,22 +146,7 @@ def async_create_expense_groups(workspace_id: int, fund_source: List[str], task_ workspace.save() - expense_objects = Expense.create_expense_objects(expenses, workspace_id) - expense_filters = ExpenseFilter.objects.filter(workspace_id=workspace_id).order_by('rank') - filtered_expenses = expense_objects - if expense_filters: - expenses_object_ids = [expense_object.id for expense_object in expense_objects] - final_query = construct_expense_filter_query(expense_filters) - mark_expenses_as_skipped(final_query, expenses_object_ids, workspace) - async_post_accounting_export_summary(workspace.fyle_org_id, workspace_id) - - filtered_expenses = Expense.objects.filter(is_skipped=False, id__in=expenses_object_ids, expensegroup__isnull=True, org_id=workspace.fyle_org_id) - - ExpenseGroup.create_expense_groups_by_report_id_fund_source(filtered_expenses, workspace_id) - - task_log.status = 'COMPLETE' - - task_log.save() + group_expenses_and_save(expenses, task_log, workspace) except FyleCredential.DoesNotExist: logger.info('Fyle credentials not found %s', workspace_id) @@ -151,11 +155,7 @@ def async_create_expense_groups(workspace_id: int, fund_source: List[str], task_ task_log.save() except Exception: - error = traceback.format_exc() - task_log.detail = {'error': error} - task_log.status = 'FATAL' - task_log.save() - logger.error('Something unexpected happened workspace_id: %s %s', task_log.workspace_id, task_log.detail) + handle_import_exception(task_log) def sync_dimensions(fyle_credentials): @@ -204,3 +204,37 @@ def post_accounting_export_summary(org_id: str, workspace_id: int) -> None: 'Internal server error while posting accounting export summary to Fyle workspace_id: %s', workspace_id ) + + +def import_and_export_expenses(report_id: str, org_id: str) -> None: + """ + Import and export expenses + :param report_id: report id + :param org_id: org id + :return: None + """ + workspace = Workspace.objects.get(fyle_org_id=org_id) + fyle_credentials = FyleCredential.objects.get(workspace_id=workspace.id) + expense_group_settings = ExpenseGroupSettings.objects.get(workspace_id=workspace.id) + + try: + with transaction.atomic(): + task_log, _ = TaskLog.objects.update_or_create(workspace_id=workspace.id, type='FETCHING_EXPENSES', defaults={'status': 'IN_PROGRESS'}) + + fund_source = get_fund_source(workspace.id) + source_account_type = get_source_account_type(fund_source) + filter_credit_expenses = get_filter_credit_expenses(expense_group_settings) + + platform = PlatformConnector(fyle_credentials) + expenses = platform.expenses.get( + source_account_type, + filter_credit_expenses=filter_credit_expenses, + report_id=report_id + ) + + group_expenses_and_save(expenses, task_log, workspace) + + export_to_qbo(workspace.id, 'AUTO') + + except Exception: + handle_import_exception(task_log) diff --git a/apps/fyle/urls.py b/apps/fyle/urls.py index 793e5f32..eaa11bcb 100644 --- a/apps/fyle/urls.py +++ b/apps/fyle/urls.py @@ -27,6 +27,7 @@ ExportableExpenseGroupsView, RefreshFyleDimensionView, SyncFyleDimensionView, + ExportView ) urlpatterns = [ @@ -41,4 +42,5 @@ path('expense_filters/', ExpenseFilterView.as_view(), name='expense-filters'), path('expenses/', ExpenseView.as_view(), name='expenses'), path('custom_fields/', CustomFieldView.as_view(), name='custom-field'), + path('exports/', ExportView.as_view(), name='exports') ] diff --git a/apps/fyle/views.py b/apps/fyle/views.py index c847179a..94a232f4 100644 --- a/apps/fyle/views.py +++ b/apps/fyle/views.py @@ -11,7 +11,7 @@ get_expense_fields, get_expense_group_ids, refresh_fyle_dimension, - sync_fyle_dimensions, + sync_fyle_dimensions ) from apps.fyle.models import Expense, ExpenseFilter, ExpenseGroup, ExpenseGroupSettings from apps.fyle.serializers import ( @@ -24,6 +24,8 @@ from apps.fyle.tasks import async_create_expense_groups, get_task_log_and_fund_source from fyle_qbo_api.utils import LookupFieldMixin +from .queue import async_import_and_export_expenses + logger = logging.getLogger(__name__) logger.level = logging.INFO @@ -93,6 +95,16 @@ def get(self, request, *args, **kwargs): return Response(expense_fields, status=status.HTTP_200_OK) +class ExportView(generics.CreateAPIView): + """ + Export View + """ + def post(self, request, *args, **kwargs): + async_import_and_export_expenses(request.data) + + return Response(data={}, status=status.HTTP_200_OK) + + class SyncFyleDimensionView(generics.ListCreateAPIView): """ Sync Fyle Dimensions View diff --git a/apps/workspaces/actions.py b/apps/workspaces/actions.py index caa8969d..29fd1108 100644 --- a/apps/workspaces/actions.py +++ b/apps/workspaces/actions.py @@ -16,9 +16,16 @@ from rest_framework.views import status from apps.fyle.helpers import get_cluster_domain, post_request -from apps.fyle.models import ExpenseGroupSettings +from apps.fyle.models import ExpenseGroupSettings, ExpenseGroup +from apps.quickbooks_online.queue import ( + schedule_bills_creation, + schedule_cheques_creation, + schedule_credit_card_purchase_creation, + schedule_journal_entry_creation, + schedule_qbo_expense_creation, +) from apps.quickbooks_online.utils import QBOConnector -from apps.workspaces.models import FyleCredential, LastExportDetail, QBOCredential, Workspace +from apps.workspaces.models import FyleCredential, LastExportDetail, QBOCredential, Workspace, WorkspaceGeneralSettings from apps.workspaces.serializers import QBOCredentialSerializer from apps.workspaces.signals import post_delete_qbo_connection from apps.workspaces.utils import assert_valid @@ -220,3 +227,84 @@ def post_to_integration_settings(workspace_id: int, active: bool): post_request(url, json.dumps(payload), refresh_token) except Exception as error: logger.error(error) + + +def export_to_qbo(workspace_id, export_mode=None): + general_settings = WorkspaceGeneralSettings.objects.get(workspace_id=workspace_id) + last_export_detail = LastExportDetail.objects.get(workspace_id=workspace_id) + last_exported_at = datetime.now() + is_expenses_exported = False + export_mode = export_mode or 'MANUAL' + + if general_settings.reimbursable_expenses_object: + + expense_group_ids = ExpenseGroup.objects.filter(fund_source='PERSONAL', exported_at__isnull=True, workspace_id=workspace_id).values_list('id', flat=True) + + if len(expense_group_ids): + is_expenses_exported = True + + if general_settings.reimbursable_expenses_object == 'BILL': + schedule_bills_creation( + workspace_id=workspace_id, + expense_group_ids=expense_group_ids, + is_auto_export=export_mode == 'AUTO' + ) + + elif general_settings.reimbursable_expenses_object == 'EXPENSE': + schedule_qbo_expense_creation( + workspace_id=workspace_id, + expense_group_ids=expense_group_ids, + is_auto_export=export_mode == 'AUTO' + ) + + elif general_settings.reimbursable_expenses_object == 'CHECK': + schedule_cheques_creation( + workspace_id=workspace_id, + expense_group_ids=expense_group_ids, + is_auto_export=export_mode == 'AUTO' + ) + + elif general_settings.reimbursable_expenses_object == 'JOURNAL ENTRY': + schedule_journal_entry_creation( + workspace_id=workspace_id, + expense_group_ids=expense_group_ids, + is_auto_export=export_mode == 'AUTO' + ) + + if general_settings.corporate_credit_card_expenses_object: + expense_group_ids = ExpenseGroup.objects.filter(fund_source='CCC', exported_at__isnull=True, workspace_id=workspace_id).values_list('id', flat=True) + + if len(expense_group_ids): + is_expenses_exported = True + + if general_settings.corporate_credit_card_expenses_object == 'JOURNAL ENTRY': + schedule_journal_entry_creation( + workspace_id=workspace_id, + expense_group_ids=expense_group_ids, + is_auto_export=export_mode == 'AUTO' + ) + + elif general_settings.corporate_credit_card_expenses_object == 'CREDIT CARD PURCHASE': + schedule_credit_card_purchase_creation( + workspace_id=workspace_id, + expense_group_ids=expense_group_ids, + is_auto_export=export_mode == 'AUTO' + ) + + elif general_settings.corporate_credit_card_expenses_object == 'DEBIT CARD EXPENSE': + schedule_qbo_expense_creation( + workspace_id=workspace_id, + expense_group_ids=expense_group_ids, + is_auto_export=export_mode == 'AUTO' + ) + + elif general_settings.corporate_credit_card_expenses_object == 'BILL': + schedule_bills_creation( + workspace_id=workspace_id, + expense_group_ids=expense_group_ids, + is_auto_export=export_mode == 'AUTO' + ) + if is_expenses_exported: + last_export_detail.last_exported_at = last_exported_at + last_export_detail.export_mode = export_mode + last_export_detail.save() diff --git a/apps/workspaces/tasks.py b/apps/workspaces/tasks.py index 35b3b01b..ecb0d398 100644 --- a/apps/workspaces/tasks.py +++ b/apps/workspaces/tasks.py @@ -11,19 +11,10 @@ from fyle_accounting_mappings.models import ExpenseAttribute from fyle_integrations_platform_connector import PlatformConnector -from apps.fyle.models import ExpenseGroup from apps.fyle.tasks import async_create_expense_groups -from apps.quickbooks_online.queue import ( - schedule_bills_creation, - schedule_cheques_creation, - schedule_credit_card_purchase_creation, - schedule_journal_entry_creation, - schedule_qbo_expense_creation, -) from apps.tasks.models import Error, TaskLog from apps.workspaces.models import ( FyleCredential, - LastExportDetail, QBOCredential, Workspace, WorkspaceGeneralSettings, @@ -32,6 +23,8 @@ from apps.workspaces.queue import schedule_email_notification from apps.users.models import User +from .actions import export_to_qbo + logger = logging.getLogger(__name__) logger.level = logging.INFO @@ -108,87 +101,6 @@ def run_sync_schedule(workspace_id): export_to_qbo(workspace_id, 'AUTO') -def export_to_qbo(workspace_id, export_mode=None): - general_settings = WorkspaceGeneralSettings.objects.get(workspace_id=workspace_id) - last_export_detail = LastExportDetail.objects.get(workspace_id=workspace_id) - last_exported_at = datetime.now() - is_expenses_exported = False - export_mode = export_mode or 'MANUAL' - - if general_settings.reimbursable_expenses_object: - - expense_group_ids = ExpenseGroup.objects.filter(fund_source='PERSONAL', exported_at__isnull=True, workspace_id=workspace_id).values_list('id', flat=True) - - if len(expense_group_ids): - is_expenses_exported = True - - if general_settings.reimbursable_expenses_object == 'BILL': - schedule_bills_creation( - workspace_id=workspace_id, - expense_group_ids=expense_group_ids, - is_auto_export=export_mode == 'AUTO' - ) - - elif general_settings.reimbursable_expenses_object == 'EXPENSE': - schedule_qbo_expense_creation( - workspace_id=workspace_id, - expense_group_ids=expense_group_ids, - is_auto_export=export_mode == 'AUTO' - ) - - elif general_settings.reimbursable_expenses_object == 'CHECK': - schedule_cheques_creation( - workspace_id=workspace_id, - expense_group_ids=expense_group_ids, - is_auto_export=export_mode == 'AUTO' - ) - - elif general_settings.reimbursable_expenses_object == 'JOURNAL ENTRY': - schedule_journal_entry_creation( - workspace_id=workspace_id, - expense_group_ids=expense_group_ids, - is_auto_export=export_mode == 'AUTO' - ) - - if general_settings.corporate_credit_card_expenses_object: - expense_group_ids = ExpenseGroup.objects.filter(fund_source='CCC', exported_at__isnull=True, workspace_id=workspace_id).values_list('id', flat=True) - - if len(expense_group_ids): - is_expenses_exported = True - - if general_settings.corporate_credit_card_expenses_object == 'JOURNAL ENTRY': - schedule_journal_entry_creation( - workspace_id=workspace_id, - expense_group_ids=expense_group_ids, - is_auto_export=export_mode == 'AUTO' - ) - - elif general_settings.corporate_credit_card_expenses_object == 'CREDIT CARD PURCHASE': - schedule_credit_card_purchase_creation( - workspace_id=workspace_id, - expense_group_ids=expense_group_ids, - is_auto_export=export_mode == 'AUTO' - ) - - elif general_settings.corporate_credit_card_expenses_object == 'DEBIT CARD EXPENSE': - schedule_qbo_expense_creation( - workspace_id=workspace_id, - expense_group_ids=expense_group_ids, - is_auto_export=export_mode == 'AUTO' - ) - - elif general_settings.corporate_credit_card_expenses_object == 'BILL': - schedule_bills_creation( - workspace_id=workspace_id, - expense_group_ids=expense_group_ids, - is_auto_export=export_mode == 'AUTO' - ) - if is_expenses_exported: - last_export_detail.last_exported_at = last_exported_at - last_export_detail.export_mode = export_mode - last_export_detail.save() - - def run_email_notification(workspace_id): expense_data = [] expense_html = '' diff --git a/apps/workspaces/views.py b/apps/workspaces/views.py index 5af3a07e..1ec90f52 100644 --- a/apps/workspaces/views.py +++ b/apps/workspaces/views.py @@ -25,7 +25,7 @@ WorkSpaceGeneralSettingsSerializer, WorkspaceSerializer, ) -from apps.workspaces.tasks import export_to_qbo +from apps.workspaces.actions import export_to_qbo from apps.workspaces.utils import generate_qbo_refresh_token logger = logging.getLogger(__name__) diff --git a/requirements.txt b/requirements.txt index e8bfb12a..77413a53 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ enum34==1.1.10 future==0.18.2 fyle==0.33.1 fyle-accounting-mappings==1.26.1 -fyle-integrations-platform-connector==1.33.0 +fyle-integrations-platform-connector==1.34.0 fyle-rest-auth==1.5.0 gevent==22.10.2 gunicorn==20.1.0