diff --git a/apps/fyle/helpers.py b/apps/fyle/helpers.py index d9ae92ad..08cf8860 100644 --- a/apps/fyle/helpers.py +++ b/apps/fyle/helpers.py @@ -1,7 +1,17 @@ import json +import traceback +import logging +from typing import List import requests from django.conf import settings +from apps.fyle.models import ExpenseGroupSettings +from apps.tasks.models import TaskLog +from apps.workspaces.models import WorkspaceGeneralSettings + +logger = logging.getLogger(__name__) + +SOURCE_ACCOUNT_MAP = {'PERSONAL': 'PERSONAL_CASH_ACCOUNT', 'CCC': 'PERSONAL_CORPORATE_CREDIT_CARD_ACCOUNT'} def post_request(url, body, refresh_token=None): @@ -24,6 +34,61 @@ def post_request(url, body, refresh_token=None): raise Exception(response.text) +def get_source_account_type(fund_source: List[str]) -> List[str]: + """ + Get source account type + :param fund_source: fund source + :return: source account type + """ + source_account_type = [] + for source in fund_source: + source_account_type.append(SOURCE_ACCOUNT_MAP[source]) + + return source_account_type + + +def get_filter_credit_expenses(expense_group_settings: ExpenseGroupSettings) -> bool: + """ + Get filter credit expenses + :param expense_group_settings: expense group settings + :return: filter credit expenses + """ + filter_credit_expenses = True + if expense_group_settings.import_card_credits: + filter_credit_expenses = False + + return filter_credit_expenses + + +def get_fund_source(workspace_id: int) -> List[str]: + """ + Get fund source + :param workspace_id: workspace id + :return: fund source + """ + general_settings = WorkspaceGeneralSettings.objects.get(workspace_id=workspace_id) + fund_source = [] + if general_settings.reimbursable_expenses_object: + fund_source.append('PERSONAL') + if general_settings.corporate_credit_card_expenses_object: + fund_source.append('CCC') + + return fund_source + + +def handle_import_exception(task_log: TaskLog) -> None: + """ + Handle import exception + :param task_log: task log + :return: None + """ + error = traceback.format_exc() + task_log.detail = {'error': error} + task_log.status = 'FATAL' + task_log.save() + logger.error('Something unexpected happened workspace_id: %s %s', task_log.workspace_id, task_log.detail) + + def get_request(url, params, refresh_token): """ Create a HTTP get request. diff --git a/apps/fyle/models.py b/apps/fyle/models.py index 2c9ba1f6..9e1cdd31 100644 --- a/apps/fyle/models.py +++ b/apps/fyle/models.py @@ -212,6 +212,7 @@ def create_expense_objects(expenses: List[Dict], workspace_id: int): "billable": expense["billable"] if expense["billable"] else False, + 'workspace_id': workspace_id }, ) diff --git a/apps/fyle/queue.py b/apps/fyle/queue.py new file mode 100644 index 00000000..1622d9e4 --- /dev/null +++ b/apps/fyle/queue.py @@ -0,0 +1,13 @@ +from django_q.tasks import async_task + + +def async_import_and_export_expenses(body: dict) -> None: + """ + Async'ly import and export expenses + :param body: body + :return: None + """ + if body.get('action') == 'ACCOUNTING_EXPORT_INITIATED' and body.get('data'): + report_id = body['data']['id'] + org_id = body['data']['org_id'] + async_task('apps.fyle.tasks.import_and_export_expenses', report_id, org_id) diff --git a/apps/fyle/tasks.py b/apps/fyle/tasks.py index 00544056..b0d69b32 100644 --- a/apps/fyle/tasks.py +++ b/apps/fyle/tasks.py @@ -1,7 +1,7 @@ import logging import traceback from datetime import datetime -from typing import List +from typing import List, Dict from django.db import transaction from fyle.platform.exceptions import InvalidTokenError as FyleInvalidTokenError @@ -11,8 +11,10 @@ from apps.tasks.enums import TaskLogStatusEnum, TaskLogTypeEnum from apps.workspaces.models import FyleCredential, Workspace, WorkspaceGeneralSettings -from .models import Expense, ExpenseGroup, ExpenseGroupSettings -from .enums import FundSourceEnum, PlatformExpensesEnum, ExpenseStateEnum +from apps.fyle.models import Expense, ExpenseGroup, ExpenseGroupSettings +from apps.fyle.enums import FundSourceEnum, PlatformExpensesEnum, ExpenseStateEnum +from apps.fyle.helpers import get_filter_credit_expenses, get_source_account_type, get_fund_source, handle_import_exception +from apps.workspaces.actions import export_to_xero logger = logging.getLogger(__name__) @@ -159,3 +161,64 @@ def async_create_expense_groups( def sync_dimensions(fyle_credentials): platform = PlatformConnector(fyle_credentials) platform.import_fyle_dimensions() + + +def group_expenses_and_save(expenses: List[Dict], task_log: TaskLog, workspace: Workspace): + expense_objects = Expense.create_expense_objects(expenses, workspace.id) + configuration: WorkspaceGeneralSettings = WorkspaceGeneralSettings.objects.get(workspace_id=workspace.id) + filtered_expenses = expense_objects + expenses_object_ids = [expense_object.id for expense_object in expense_objects] + + filtered_expenses = Expense.objects.filter( + is_skipped=False, + id__in=expenses_object_ids, + expensegroup__isnull=True, + org_id=workspace.fyle_org_id + ) + + ExpenseGroup.create_expense_groups_by_report_id_fund_source( + filtered_expenses, configuration, workspace.id + ) + + task_log.status = 'COMPLETE' + task_log.save() + + +def import_and_export_expenses(report_id: str, org_id: str) -> None: + """ + Import and export expenses + :param report_id: report id + :param org_id: org id + :return: None + """ + workspace = Workspace.objects.get(fyle_org_id=org_id) + fyle_credentials = FyleCredential.objects.get(workspace_id=workspace.id) + expense_group_settings = ExpenseGroupSettings.objects.get(workspace_id=workspace.id) + + try: + with transaction.atomic(): + task_log, _ = TaskLog.objects.update_or_create(workspace_id=workspace.id, type='FETCHING_EXPENSES', defaults={'status': 'IN_PROGRESS'}) + + fund_source = get_fund_source(workspace.id) + source_account_type = get_source_account_type(fund_source) + filter_credit_expenses = get_filter_credit_expenses(expense_group_settings) + + platform = PlatformConnector(fyle_credentials) + expenses = platform.expenses.get( + source_account_type, + filter_credit_expenses=filter_credit_expenses, + report_id=report_id + ) + + group_expenses_and_save(expenses, task_log, workspace) + + # Export only selected expense groups + expense_ids = Expense.objects.filter(report_id=report_id, org_id=org_id).values_list('id', flat=True) + expense_groups = ExpenseGroup.objects.filter(expenses__id__in=[expense_ids], workspace_id=workspace.id).distinct('id').values('id') + expense_group_ids = [expense_group['id'] for expense_group in expense_groups] + + if len(expense_group_ids): + export_to_xero(workspace.id, None, expense_group_ids) + + except Exception: + handle_import_exception(task_log) diff --git a/apps/fyle/urls.py b/apps/fyle/urls.py index a4051bce..153dc610 100644 --- a/apps/fyle/urls.py +++ b/apps/fyle/urls.py @@ -8,6 +8,7 @@ ExportableExpenseGroupsView, RefreshFyleDimensionView, SyncFyleDimensionView, + ExportView ) urlpatterns = [ @@ -26,4 +27,5 @@ path("expense_group_settings/", ExpenseGroupSettingsView.as_view()), path("sync_dimensions/", SyncFyleDimensionView.as_view()), path("refresh_dimensions/", RefreshFyleDimensionView.as_view()), + path('exports/', ExportView.as_view(), name='exports') ] diff --git a/apps/fyle/views.py b/apps/fyle/views.py index 029fb847..b00076cd 100644 --- a/apps/fyle/views.py +++ b/apps/fyle/views.py @@ -8,6 +8,7 @@ from apps.fyle.serializers import ExpenseFieldSerializer, ExpenseGroupSerializer, ExpenseGroupSettingsSerializer from apps.fyle.tasks import async_create_expense_groups, get_task_log_and_fund_source from fyle_xero_api.utils import LookupFieldMixin +from apps.fyle.queue import async_import_and_export_expenses class ExpenseGroupView(LookupFieldMixin, generics.ListCreateAPIView): @@ -106,3 +107,16 @@ def get(self, request, *args, **kwargs): data={"exportable_expense_group_ids": expense_group_ids}, status=status.HTTP_200_OK, ) + + +class ExportView(generics.CreateAPIView): + """ + Export View + """ + authentication_classes = [] + permission_classes = [] + + def post(self, request, *args, **kwargs): + async_import_and_export_expenses(request.data) + + return Response(data={}, status=status.HTTP_200_OK) diff --git a/apps/workspaces/actions.py b/apps/workspaces/actions.py index 5f8fc4b5..a9a1a334 100644 --- a/apps/workspaces/actions.py +++ b/apps/workspaces/actions.py @@ -1,4 +1,5 @@ import logging +from datetime import datetime, timedelta from django.contrib.auth import get_user_model from django.core.cache import cache @@ -11,10 +12,13 @@ from apps.fyle.helpers import get_cluster_domain from apps.fyle.models import ExpenseGroupSettings from apps.mappings.models import TenantMapping -from apps.workspaces.models import FyleCredential, LastExportDetail, Workspace, XeroCredentials +from apps.workspaces.models import FyleCredential, LastExportDetail, Workspace, XeroCredentials, WorkspaceGeneralSettings, WorkspaceSchedule +from apps.fyle.models import ExpenseGroup from apps.workspaces.signals import post_delete_xero_connection from apps.workspaces.utils import generate_xero_refresh_token from apps.xero.utils import XeroConnector +from apps.fyle.enums import FundSourceEnum +from apps.xero.tasks import schedule_bank_transaction_creation, schedule_bills_creation logger = logging.getLogger(__name__) @@ -170,3 +174,60 @@ def get_workspace_admin(workspace_id): {"name": employee.detail["full_name"], "email": admin.email} ) return admin_email + + +def export_to_xero(workspace_id, export_mode="MANUAL", expense_group_ids=[]): + general_settings = WorkspaceGeneralSettings.objects.get(workspace_id=workspace_id) + last_export_detail = LastExportDetail.objects.get(workspace_id=workspace_id) + workspace_schedule = WorkspaceSchedule.objects.filter(workspace_id=workspace_id, interval_hours__gt=0, enabled=True).first() + + last_exported_at = datetime.now() + is_expenses_exported = False + export_mode = export_mode or 'MANUAL' + expense_group_filters = { + 'exported_at__isnull': True, + 'workspace_id': workspace_id + } + if expense_group_ids: + expense_group_filters['id__in'] = expense_group_ids + + if general_settings.reimbursable_expenses_object: + expense_group_ids = ExpenseGroup.objects.filter( + fund_source=FundSourceEnum.PERSONAL, + **expense_group_filters + ).values_list("id", flat=True) + + if len(expense_group_ids): + is_expenses_exported = True + + schedule_bills_creation( + workspace_id=workspace_id, + expense_group_ids=expense_group_ids, + is_auto_export=export_mode == 'AUTO', + fund_source='PERSONAL' + ) + + if general_settings.corporate_credit_card_expenses_object: + expense_group_ids = ExpenseGroup.objects.filter( + fund_source=FundSourceEnum.CCC, + **expense_group_filters + ).values_list("id", flat=True) + + if len(expense_group_ids): + is_expenses_exported = True + + schedule_bank_transaction_creation( + workspace_id=workspace_id, + expense_group_ids=expense_group_ids, + is_auto_export=export_mode == 'AUTO', + fund_source='CCC' + ) + + if is_expenses_exported: + last_export_detail.last_exported_at = last_exported_at + last_export_detail.export_mode = export_mode or 'MANUAL' + + if workspace_schedule: + last_export_detail.next_export_at = last_exported_at + timedelta(hours=workspace_schedule.interval_hours) + + last_export_detail.save() diff --git a/apps/workspaces/tasks.py b/apps/workspaces/tasks.py index caa1c3e1..d4d2178a 100644 --- a/apps/workspaces/tasks.py +++ b/apps/workspaces/tasks.py @@ -7,7 +7,6 @@ from fyle_rest_auth.helpers import get_fyle_admin from apps.fyle.helpers import post_request -from apps.fyle.models import ExpenseGroup from apps.fyle.tasks import async_create_expense_groups from apps.fyle.enums import FundSourceEnum @@ -19,9 +18,8 @@ from apps.users.models import User from apps.workspaces.email import get_admin_name, get_errors, get_failed_task_logs_count, send_failure_notification_email -from apps.workspaces.models import FyleCredential, LastExportDetail, Workspace, WorkspaceGeneralSettings, WorkspaceSchedule - -from apps.xero.tasks import create_chain_and_export, schedule_bank_transaction_creation, schedule_bills_creation +from apps.workspaces.models import FyleCredential, Workspace, WorkspaceGeneralSettings, WorkspaceSchedule +from apps.workspaces.actions import export_to_xero logger = logging.getLogger(__name__) @@ -57,37 +55,6 @@ def run_sync_schedule(workspace_id): export_to_xero(workspace_id, "AUTO") -def export_to_xero(workspace_id, export_mode="MANUAL"): - general_settings = WorkspaceGeneralSettings.objects.get(workspace_id=workspace_id) - last_export_detail = LastExportDetail.objects.get(workspace_id=workspace_id) - last_exported_at = datetime.now() - chaining_attributes = [] - - if general_settings.reimbursable_expenses_object: - expense_group_ids = ExpenseGroup.objects.filter( - fund_source=FundSourceEnum.PERSONAL, - workspace_id=workspace_id - ).values_list("id", flat=True) - chaining_attributes.extend( - schedule_bills_creation(workspace_id, expense_group_ids) - ) - - if general_settings.corporate_credit_card_expenses_object: - expense_group_ids = ExpenseGroup.objects.filter( - fund_source=FundSourceEnum.CCC, - workspace_id=workspace_id - ).values_list("id", flat=True) - chaining_attributes.extend( - schedule_bank_transaction_creation(workspace_id, expense_group_ids) - ) - - if chaining_attributes: - create_chain_and_export(chaining_attributes, workspace_id) - last_export_detail.last_exported_at = last_exported_at - last_export_detail.export_mode = export_mode - last_export_detail.save() - - def async_update_fyle_credentials(fyle_org_id: str, refresh_token: str): fyle_credentials = FyleCredential.objects.filter( workspace__fyle_org_id=fyle_org_id diff --git a/apps/workspaces/views.py b/apps/workspaces/views.py index f4c4fc2d..048e6b17 100644 --- a/apps/workspaces/views.py +++ b/apps/workspaces/views.py @@ -18,7 +18,7 @@ WorkspaceSerializer, XeroCredentialSerializer, ) -from apps.workspaces.tasks import export_to_xero +from apps.workspaces.actions import export_to_xero from apps.workspaces.utils import generate_xero_identity logger = logging.getLogger(__name__) diff --git a/apps/xero/tasks.py b/apps/xero/tasks.py index e1ba9c11..4c34310a 100644 --- a/apps/xero/tasks.py +++ b/apps/xero/tasks.py @@ -268,46 +268,36 @@ def create_bill( ) -def create_chain_and_export(chaining_attributes: list, workspace_id: int) -> None: +def __create_chain_and_run(fyle_credentials: FyleCredential, xero_connection, in_progress_expenses: List[Expense], + workspace_id: int, chain_tasks: List[dict], fund_source: str) -> None: """ - Create a chain of expense groups and export them to Xero - :param chaining_attributes: - :param workspace_id: + Create chain and run + :param fyle_credentials: Fyle credentials + :param in_progress_expenses: List of in progress expenses + :param workspace_id: workspace id + :param chain_tasks: List of chain tasks + :param fund_source: Fund source :return: None """ - try: - xero_credentials = XeroCredentials.get_active_xero_credentials(workspace_id) - xero_connection = XeroConnector(xero_credentials, workspace_id) - except (UnsuccessfulAuthentication, XeroCredentials.DoesNotExist): - xero_connection = None - chain = Chain() - - fyle_credentials = FyleCredential.objects.get(workspace_id=workspace_id) chain.append("apps.fyle.tasks.sync_dimensions", fyle_credentials) - for group in chaining_attributes: - trigger_function = "apps.xero.tasks.create_{}".format(group["export_type"]) - chain.append( - trigger_function, - group["expense_group_id"], - group["task_log_id"], - xero_connection, - group["last_export"] - ) + # chain.append('apps.netsuite.tasks.update_expense_and_post_summary', in_progress_expenses, workspace_id, fund_source) + + for task in chain_tasks: + chain.append(task['target'], task['expense_group_id'], task['task_log_id'], xero_connection, task['last_export']) - if chain.length() > 1: - chain.run() + # chain.append('apps.fyle.tasks.post_accounting_export_summary', fyle_credentials.workspace.fyle_org_id, workspace_id, fund_source) + chain.run() -def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str]) -> list: +def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str) -> list: """ Schedule bills creation :param expense_group_ids: List of expense group ids :param workspace_id: workspace id :return: List of chaining attributes """ - chaining_attributes = [] if expense_group_ids: expense_groups = ExpenseGroup.objects.filter( Q(tasklog__id__isnull=True) @@ -318,6 +308,9 @@ def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str]) -> exported_at__isnull=True, ).all() + chain_tasks = [] + in_progress_expenses = [] + for index, expense_group in enumerate(expense_groups): task_log, _ = TaskLog.objects.get_or_create( workspace_id=expense_group.workspace_id, @@ -332,18 +325,23 @@ def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str]) -> if expense_groups.count() == index + 1: last_export = True - chaining_attributes.append( - { - "expense_group_id": expense_group.id, - "task_log_id": task_log.id, - "export_type": "bill", - "last_export": last_export, - } - ) + chain_tasks.append({ + 'target': 'apps.xero.tasks.create_bill', + 'expense_group_id': expense_group.id, + 'task_log_id': task_log.id, + 'last_export': last_export}) - task_log.save() + if not (is_auto_export and expense_group.expenses.first().previous_export_state == 'ERROR'): + in_progress_expenses.extend(expense_group.expenses.all()) - return chaining_attributes + if len(chain_tasks) > 0: + fyle_credentials = FyleCredential.objects.get(workspace_id=workspace_id) + try: + xero_credentials = XeroCredentials.get_active_xero_credentials(workspace_id) + xero_connection = XeroConnector(xero_credentials, workspace_id) + except (UnsuccessfulAuthentication, XeroCredentials.DoesNotExist): + xero_connection = None + __create_chain_and_run(fyle_credentials, xero_connection, in_progress_expenses, workspace_id, chain_tasks, fund_source) def get_linked_transaction_object(export_instance, line_items: list): @@ -518,7 +516,7 @@ def create_bank_transaction( def schedule_bank_transaction_creation( - workspace_id: int, expense_group_ids: List[str] + workspace_id: int, expense_group_ids: List[str], is_auto_export: bool, fund_source: str ) -> list: """ Schedule bank transaction creation @@ -526,7 +524,6 @@ def schedule_bank_transaction_creation( :param workspace_id: workspace id :return: List of chaining attributes """ - chaining_attributes = [] if expense_group_ids: expense_groups = ExpenseGroup.objects.filter( Q(tasklog__id__isnull=True) @@ -537,6 +534,9 @@ def schedule_bank_transaction_creation( exported_at__isnull=True, ).all() + chain_tasks = [] + in_progress_expenses = [] + for index, expense_group in enumerate(expense_groups): task_log, _ = TaskLog.objects.get_or_create( workspace_id=expense_group.workspace_id, @@ -551,18 +551,23 @@ def schedule_bank_transaction_creation( if expense_groups.count() == index + 1: last_export = True - chaining_attributes.append( - { - "expense_group_id": expense_group.id, - "task_log_id": task_log.id, - "export_type": "bank_transaction", - "last_export": last_export, - } - ) - - task_log.save() - - return chaining_attributes + chain_tasks.append({ + 'target': 'apps.xero.tasks.create_bank_transaction', + 'expense_group_id': expense_group.id, + 'task_log_id': task_log.id, + 'last_export': last_export}) + + if not (is_auto_export and expense_group.expenses.first().previous_export_state == 'ERROR'): + in_progress_expenses.extend(expense_group.expenses.all()) + + if len(chain_tasks) > 0: + fyle_credentials = FyleCredential.objects.get(workspace_id=workspace_id) + try: + xero_credentials = XeroCredentials.get_active_xero_credentials(workspace_id) + xero_connection = XeroConnector(xero_credentials, workspace_id) + except (UnsuccessfulAuthentication, XeroCredentials.DoesNotExist): + xero_connection = None + __create_chain_and_run(fyle_credentials, xero_connection, in_progress_expenses, workspace_id, chain_tasks, fund_source) def __validate_expense_group(expense_group: ExpenseGroup): diff --git a/tests/test_xero/test_tasks.py b/tests/test_xero/test_tasks.py index e818bd30..b2416954 100644 --- a/tests/test_xero/test_tasks.py +++ b/tests/test_xero/test_tasks.py @@ -20,7 +20,6 @@ check_xero_object_status, create_bank_transaction, create_bill, - create_chain_and_export, create_missing_currency, create_or_update_employee_mapping, create_payment, @@ -412,10 +411,9 @@ def test_schedule_bills_creation(db): task_log.status = "READY" task_log.save() - chaining_attributes = schedule_bills_creation( - workspace_id=workspace_id, expense_group_ids=[4] + schedule_bills_creation( + workspace_id=workspace_id, expense_group_ids=[4], is_auto_export=False, fund_source="PERSONAL" ) - assert len(chaining_attributes) == 1 def test_post_create_bank_transaction_success(mocker, db): @@ -511,10 +509,9 @@ def test_schedule_bank_transaction_creation(db): task_log.status = "READY" task_log.save() - chaining_attributes = schedule_bank_transaction_creation( - workspace_id=workspace_id, expense_group_ids=[5] + schedule_bank_transaction_creation( + workspace_id=workspace_id, expense_group_ids=[5], is_auto_export=False, fund_source="CCC" ) - assert len(chaining_attributes) == 1 def test_create_bank_transactions_exceptions(db): @@ -932,20 +929,6 @@ def test_update_xero_short_code(db, mocker): update_xero_short_code(workspace_id) -def test_create_chain_and_export(db): - workspace_id = 1 - - chaining_attributes = [ - { - "expense_group_id": 4, - "export_type": "bill", - "task_log_id": 3, - "last_export": False, - } - ] - create_chain_and_export(chaining_attributes, workspace_id) - - def test_update_last_export_details(db): workspace_id = 1