From ed81a6448dc151fcdba64750f1e60abbcc46c286 Mon Sep 17 00:00:00 2001 From: Hrishabh Tiwari <74908943+Hrishabh17@users.noreply.github.com> Date: Wed, 26 Jun 2024 18:31:42 +0530 Subject: [PATCH] Add support for editing expenses (#606) * Add support for editing expenses * change log --- apps/exceptions.py | 97 +++++++++++++ apps/fyle/helpers.py | 13 +- apps/fyle/queue.py | 14 +- apps/fyle/tasks.py | 26 ++++ apps/fyle/views.py | 17 ++- requirements.txt | 2 +- tests/test_fyle/fixtures.py | 266 ++++++++++++++++++++++++++++++++++ tests/test_fyle/test_queue.py | 8 +- tests/test_fyle/test_tasks.py | 67 ++++++++- 9 files changed, 492 insertions(+), 18 deletions(-) create mode 100644 apps/exceptions.py diff --git a/apps/exceptions.py b/apps/exceptions.py new file mode 100644 index 00000000..0684ceea --- /dev/null +++ b/apps/exceptions.py @@ -0,0 +1,97 @@ +import logging + +from fyle.platform.exceptions import InvalidTokenError as FyleInvalidTokenError +from fyle.platform.exceptions import NoPrivilegeError +from rest_framework.response import Response +from rest_framework.views import status +from rest_framework.exceptions import ValidationError + +from apps.fyle.models import ExpenseGroup +from apps.mappings.models import GeneralMapping +from apps.workspaces.models import FyleCredential, NetSuiteCredentials, Workspace, Configuration + +logger = logging.getLogger(__name__) +logger.level = logging.INFO + + +def handle_view_exceptions(): + def decorator(func): + def new_fn(*args, **kwargs): + try: + return func(*args, **kwargs) + except ExpenseGroup.DoesNotExist: + return Response( + data={'message': 'Expense group not found'}, + status=status.HTTP_400_BAD_REQUEST + ) + + except FyleCredential.DoesNotExist: + return Response( + data={'message': 'Fyle credentials not found in workspace'}, + status=status.HTTP_400_BAD_REQUEST + ) + + except FyleInvalidTokenError as exception: + logger.info( + 'Fyle token expired workspace_id - %s %s', + kwargs['workspace_id'], + {'error': exception.response} + ) + return Response( + data={'message': 'Fyle token expired workspace_id'}, + status=status.HTTP_400_BAD_REQUEST + ) + + except GeneralMapping.DoesNotExist: + return Response( + {'message': 'General mappings do not exist for the workspace'}, + status=status.HTTP_400_BAD_REQUEST + ) + + except NoPrivilegeError as exception: + logger.info( + 'Invalid Fyle Credentials / Admin is disabled for workspace_id%s %s', + kwargs['workspace_id'], + {'error': exception.response} + ) + return Response( + data={'message': 'Invalid Fyle Credentials / Admin is disabled'}, + status=status.HTTP_400_BAD_REQUEST + ) + + except Workspace.DoesNotExist: + return Response( + data={'message': 'Workspace with this id does not exist'}, + status=status.HTTP_400_BAD_REQUEST + ) + + except Configuration.DoesNotExist: + return Response( + data={'message': 'Configuration does not exist in workspace'}, + status=status.HTTP_400_BAD_REQUEST + ) + + except NetSuiteCredentials.DoesNotExist: + logger.info('Netsuite credentials not found in workspace') + return Response( + data={'message': 'Netsuite credentials not found in workspace'}, + status=status.HTTP_400_BAD_REQUEST + ) + + except ValidationError as e: + logger.exception(e) + return Response( + {"message": e.detail}, + status=status.HTTP_400_BAD_REQUEST + ) + + except Exception as exception: + logger.exception(exception) + return Response( + data={'message': 'An unhandled error has occurred, please re-try later'}, + status=status.HTTP_400_BAD_REQUEST + ) + + return new_fn + + return decorator diff --git a/apps/fyle/helpers.py b/apps/fyle/helpers.py index 5ed46fe5..412866c3 100644 --- a/apps/fyle/helpers.py +++ b/apps/fyle/helpers.py @@ -1,12 +1,13 @@ import json +import logging import traceback import requests from datetime import datetime, timezone from fyle_integrations_platform_connector import PlatformConnector -import logging from django.conf import settings from django.db.models import Q +from rest_framework.exceptions import ValidationError from fyle_accounting_mappings.models import ExpenseAttribute from apps.fyle.models import ExpenseGroupSettings, ExpenseFilter, ExpenseGroup, Expense @@ -397,6 +398,16 @@ def construct_expense_filter(expense_filter: ExpenseFilter): return constructed_expense_filter +def assert_valid_request(workspace_id:int, fyle_org_id:str): + """ + Assert if the request is valid by checking + the url_workspace_id and fyle_org_id workspace + """ + workspace = Workspace.objects.get(fyle_org_id=fyle_org_id) + if workspace.id != workspace_id: + raise ValidationError('Workspace mismatch') + + class AdvanceSearchFilter(django_filters.FilterSet): def filter_queryset(self, queryset): or_filtered_queryset = queryset.none() diff --git a/apps/fyle/queue.py b/apps/fyle/queue.py index 68cec648..b9f62096 100644 --- a/apps/fyle/queue.py +++ b/apps/fyle/queue.py @@ -1,4 +1,9 @@ +import logging from django_q.tasks import async_task +from apps.fyle.helpers import assert_valid_request + +logger = logging.getLogger(__name__) +logger.level = logging.INFO def async_post_accounting_export_summary(org_id: str, workspace_id: int) -> None: @@ -12,7 +17,7 @@ def async_post_accounting_export_summary(org_id: str, workspace_id: int) -> None async_task('apps.fyle.tasks.post_accounting_export_summary', org_id, workspace_id) -def async_import_and_export_expenses(body: dict) -> None: +def async_import_and_export_expenses(body: dict, workspace_id: int) -> None: """ Async'ly import and export expenses :param body: body @@ -21,4 +26,11 @@ def async_import_and_export_expenses(body: dict) -> None: if body.get('action') == 'ACCOUNTING_EXPORT_INITIATED' and body.get('data'): report_id = body['data']['id'] org_id = body['data']['org_id'] + assert_valid_request(workspace_id=workspace_id, fyle_org_id=org_id) async_task('apps.fyle.tasks.import_and_export_expenses', report_id, org_id) + + elif body.get('action') == 'UPDATED_AFTER_APPROVAL' and body.get('data') and body.get('resource') == 'EXPENSE': + org_id = body['data']['org_id'] + logger.info("| Updating non-exported expenses through webhook | Content: {{WORKSPACE_ID: {} Payload: {}}}".format(workspace_id, body.get('data'))) + assert_valid_request(workspace_id=workspace_id, fyle_org_id=org_id) + async_task('apps.fyle.tasks.update_non_exported_expenses', body['data']) \ No newline at end of file diff --git a/apps/fyle/tasks.py b/apps/fyle/tasks.py index 614b38ee..47603529 100644 --- a/apps/fyle/tasks.py +++ b/apps/fyle/tasks.py @@ -7,6 +7,7 @@ from django_q.tasks import async_task from fyle_integrations_platform_connector import PlatformConnector +from fyle_integrations_platform_connector.apis.expenses import Expenses as FyleExpenses from fyle.platform.exceptions import ( RetryException, InternalServerError, @@ -293,3 +294,28 @@ def import_and_export_expenses(report_id: str, org_id: str) -> None: except Exception: handle_import_exception(task_log) + + +def update_non_exported_expenses(data: Dict) -> None: + """ + To update expenses not in COMPLETE, IN_PROGRESS state + """ + expense_state = None + org_id = data['org_id'] + expense_id = data['id'] + workspace = Workspace.objects.get(fyle_org_id=org_id) + expense = Expense.objects.filter(workspace_id=workspace.id, expense_id=expense_id).first() + + if expense: + if 'state' in expense.accounting_export_summary: + expense_state = expense.accounting_export_summary['state'] + else: + expense_state = 'NOT_EXPORTED' + + if expense_state and expense_state not in ['COMPLETE', 'IN_PROGRESS']: + expense_obj = [] + expense_obj.append(data) + expense_objects = FyleExpenses().construct_expense_object(expense_obj, expense.workspace_id) + Expense.create_expense_objects( + expense_objects, expense.workspace_id + ) diff --git a/apps/fyle/views.py b/apps/fyle/views.py index 300e4931..c09e0872 100644 --- a/apps/fyle/views.py +++ b/apps/fyle/views.py @@ -13,7 +13,7 @@ from fyle_accounting_mappings.models import ExpenseAttribute from fyle_accounting_mappings.serializers import ExpenseAttributeSerializer -from apps.workspaces.models import Configuration, FyleCredential, Workspace +from apps.workspaces.models import FyleCredential, Workspace from fyle_netsuite_api.utils import LookupFieldMixin from .tasks import schedule_expense_group_creation, get_task_log_and_fund_source, create_expense_groups @@ -24,8 +24,7 @@ from .queue import async_import_and_export_expenses from .constants import DEFAULT_FYLE_CONDITIONS -from fyle.platform import Platform -from fyle_netsuite_api import settings +from apps.exceptions import handle_view_exceptions from django_filters.rest_framework import DjangoFilterBackend @@ -40,6 +39,7 @@ class ExpenseGroupViewV2(LookupFieldMixin, generics.ListCreateAPIView): filter_backends = (DjangoFilterBackend,) filterset_class = ExpenseGroupSearchFilter + class ExpenseViewV2(LookupFieldMixin, generics.ListAPIView): """ Expense view @@ -281,13 +281,13 @@ def post(self, request, *args, **kwargs): }, status=status.HTTP_400_BAD_REQUEST ) - except Exception : + except Exception: return Response( data={ 'message': 'Error in syncing Dimensions' }, status=status.HTTP_400_BAD_REQUEST - ) + ) class RefreshFyleDimensionView(generics.ListCreateAPIView): @@ -344,7 +344,7 @@ def delete(self, request, *args, **kwargs): return Response(data={ 'workspace_id': workspace_id, - 'rank' : rank, + 'rank': rank, 'message': 'Expense filter deleted' }) @@ -400,7 +400,7 @@ def get(self, request, *args, **kwargs): platform = PlatformConnector(fyle_credentails) custom_fields = platform.expense_custom_fields.list_all() - response = [] + response = [] response.extend(DEFAULT_FYLE_CONDITIONS) for custom_field in custom_fields: @@ -456,7 +456,8 @@ class ExportView(generics.CreateAPIView): authentication_classes = [] permission_classes = [] + @handle_view_exceptions() def post(self, request, *args, **kwargs): - async_import_and_export_expenses(request.data) + async_import_and_export_expenses(request.data, int(kwargs['workspace_id'])) return Response(data={}, status=status.HTTP_200_OK) diff --git a/requirements.txt b/requirements.txt index 934f2f22..3846f9fb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ enum34==1.1.10 future==0.18.2 fyle==0.37.0 fyle-accounting-mappings==1.32.2 -fyle-integrations-platform-connector==1.38.0 +fyle-integrations-platform-connector==1.38.1 fyle-rest-auth==1.7.2 gunicorn==20.1.0 gevent==23.9.1 diff --git a/tests/test_fyle/fixtures.py b/tests/test_fyle/fixtures.py index 81690a29..64fb068c 100644 --- a/tests/test_fyle/fixtures.py +++ b/tests/test_fyle/fixtures.py @@ -1,4 +1,270 @@ data = { + "raw_expense": { + 'accounting_export_summary': { + 'error_type': 'ACCOUNTING_INTEGRATION_ERROR', + 'state': 'ERROR', + 'tpa_id': 'tpayfjPPHTDgv', + 'url': 'https://staging1.fyle.tech/app/settings/#/integrations/native_apps?integrationIframeTarget=integrations/intacct/main/dashboard' + }, + 'activity_details': None, + 'added_to_report_at': None, + 'admin_amount': None, + 'advance_wallet_id': None, + 'amount': 12, + 'approvals': [ + { + 'approver_user': { + 'email': 'admin1@fyleforimporrttest.in', + 'full_name': 'Theresa Brown', + 'id': 'usVN2WTtPqE7' + }, + 'approver_user_id': 'usVN2WTtPqE7', + 'state': 'APPROVAL_DONE' + } + ], + 'approver_comments': [], + 'category': { + 'code': '223', + 'display_name': 'ABN Withholding', + 'id': 317995, + 'name': 'ABN Withholding', + 'sub_category': None, + 'system_category': None + }, + 'category_id': 317995, + 'claim_amount': 12, + 'code': None, + 'commute_deduction': None, + 'commute_details': None, + 'commute_details_id': None, + 'cost_center': { + 'code': '96441', + 'id': 23166, + 'name': 'Administration' + }, + 'cost_center_id': 23166, + 'created_at': '2024-05-10T07:52:10.551260+00:00', + 'creator_user_id': 'usVN2WTtPqE7', + 'currency': 'USD', + 'custom_fields': [ + { + 'is_enabled': True, + 'name': 'Custom Expense Field', + 'type': 'TEXT', + 'value': None + }, + { + 'is_enabled': True, + 'name': 'Locationcustom', + 'type': 'SELECT', + 'value': None + }, + { + 'is_enabled': True, + 'name': 'Deptcustom', + 'type': 'SELECT', + 'value': None + } + ], + 'custom_fields_flattened': { + 'custom_expense_field': None, + 'deptcustom': None, + 'locationcustom': None + }, + 'distance': None, + 'distance_unit': None, + 'employee': { + 'business_unit': None, + 'code': None, + 'custom_fields': [], + 'department': None, + 'department_id': None, + 'flattened_custom_field': {}, + 'has_accepted_invite': True, + 'id': 'ouhC0BNdc33I', + 'is_enabled': True, + 'joined_at': None, + 'level': None, + 'location': None, + 'mobile': None, + 'org_id': 'orAW3T2QmroT', + 'org_name': 'Fyle For import_test', + 'title': None, + 'user': { + 'email': 'admin1@fyleforimporrttest.in', + 'full_name': 'Theresa Brown', + 'id': 'usVN2WTtPqE7' + }, + 'user_id': 'usVN2WTtPqE7' + }, + 'employee_id': 'ouhC0BNdc33I', + 'ended_at': None, + 'expense_rule_data': None, + 'expense_rule_id': None, + 'extracted_data': None, + 'file_ids': [], + 'files': [], + 'foreign_amount': None, + 'foreign_currency': None, + 'hotel_is_breakfast_provided': False, + 'id': 'txhJLOSKs1iN', + 'invoice_number': None, + 'is_billable': None, + 'is_corporate_card_transaction_auto_matched': False, + 'is_exported': None, + 'is_manually_flagged': None, + 'is_physical_bill_submitted': None, + 'is_policy_flagged': None, + 'is_receipt_mandatory': None, + 'is_reimbursable': True, + 'is_split': False, + 'is_verified': True, + 'is_weekend_spend': False, + 'last_exported_at': None, + 'last_settled_at': '2024-05-10T07:55:07.373278+00:00', + 'last_verified_at': '2024-05-10T07:55:02.329280+00:00', + 'locations': [], + 'matched_corporate_card_transaction_ids': [], + 'matched_corporate_card_transactions': [], + 'merchant': None, + 'mileage_calculated_amount': None, + 'mileage_calculated_distance': None, + 'mileage_is_round_trip': None, + 'mileage_rate': None, + 'mileage_rate_id': None, + 'missing_mandatory_fields': { + 'amount': False, + 'currency': False, + 'expense_field_ids': [], + 'receipt': False + }, + 'org_id': 'orAW3T2QmroT', + 'per_diem_num_days': None, + 'per_diem_rate': None, + 'per_diem_rate_id': None, + 'physical_bill_submitted_at': None, + 'policy_amount': None, + 'policy_checks': { + 'are_approvers_added': False, + 'is_amount_limit_applied': False, + 'is_flagged_ever': False, + 'violations': None + }, + 'project': { + 'code': 'B3DNLG7TVM', + 'display_name': 'Project 6', + 'id': 330241, + 'name': 'Project 6', + 'sub_project': None + }, + 'project_id': 330241, + 'purpose': None, + 'report': { + 'amount': 12, + 'approvals': [ + { + 'approver_user': { + 'email': 'admin1@fyleforimporrttest.in', + 'full_name': 'Theresa Brown', + 'id': 'usVN2WTtPqE7' + }, + 'approver_user_id': 'usVN2WTtPqE7', + 'state': 'APPROVAL_DONE' + } + ], + 'id': 'rpN41rGGnxNI', + 'last_approved_at': '2024-05-10T07:53:25.774+00:00', + 'last_paid_at': None, + 'last_submitted_at': '2024-05-10T07:53:09.457+00:00', + 'last_verified_at': '2024-05-10T07:55:02.32928+00:00', + 'reimbursement_id': 'reimYNNUkKQiWp', + 'reimbursement_seq_num': 'P/2024/05/T/P/2024/05/R/30', + 'seq_num': 'C/2024/05/R/45', + 'settlement_id': 'setUkp31alIp7', + 'state': 'PAYMENT_PROCESSING', + 'title': '#5: May 2024' + }, + 'report_id': 'rpN41rGGnxNI', + 'report_last_approved_at': '2024-05-10T07:53:25.774000+00:00', + 'report_last_paid_at': None, + 'report_settlement_id': 'setUkp31alIp7', + 'seq_num': 'E/2024/05/T/442', + 'source': 'WEBAPP', + 'source_account': { + 'id': 'accUMhoA4foa5', + 'type': 'PERSONAL_CASH_ACCOUNT' + }, + 'source_account_id': 'accUMhoA4foa5', + 'spent_at': '2024-05-10T00:00:00+00:00', + 'split_group_amount': None, + 'split_group_id': 'txhJLOSKs1iN', + 'started_at': None, + 'state': 'PAYMENT_PROCESSING', + 'state_display_name': 'Processing', + 'tax_amount': None, + 'tax_group': None, + 'tax_group_id': None, + 'travel_classes': [], + 'updated_at': '2024-06-10T11:41:40.779611+00:00', + 'user': { + 'email': 'admin1@fyleforimporrttest.in', + 'full_name': 'Theresa Brown', + 'id': 'usVN2WTtPqE7' + }, + 'user_id': 'usVN2WTtPqE7', + 'verifications': [ + { + 'verifier_user': { + 'email': 'owner@fyleforimporrttest.in', + 'full_name': 'Fyle For import_test', + 'id': 'usbzW0rVpuWC' + }, + 'verifier_user_id': 'usbzW0rVpuWC' + } + ], + 'verifier_comments': [] + }, + "default_raw_expense": { + 'employee_email': 'admin1@fyleforimporrttest.in', + 'employee_name': 'Theresa Brown', + 'category': 'Old Category', + 'sub_category': None, + 'project': 'Project 6', + 'org_id': 'orAW3T2QmroT', + 'expense_number': 'E/2024/05/T/442', + 'claim_number': 'C/2024/05/R/45', + 'amount': 12.0, + 'currency': 'USD', + 'foreign_amount': None, + 'foreign_currency': None, + 'reimbursable': True, + 'state': 'PAYMENT_PROCESSING', + 'vendor': None, + 'cost_center': 'Administration', + 'corporate_card_id': None, + 'purpose': None, + 'report_id': 'rpN41rGGnxNI', + 'billable': False, + 'file_ids': [], + 'spent_at': '2024-05-10 17:00:00', + 'approved_at': '2024-05-10 07:53:25', + 'posted_at': None, + 'is_skipped': False, + 'expense_created_at': '2024-05-10 07:52:10', + 'expense_updated_at': '2024-05-13 05:53:25', + 'fund_source': 'PERSONAL', + 'verified_at': '2024-05-10 07:53:25', + 'custom_properties': { + 'Deptcustom': None, + 'Locationcustom': None, + 'Custom Expense Field': None + }, + 'report_title': '#5: May 2024', + 'tax_amount': None, + 'tax_group_id': None, + 'previous_export_state': None, + 'accounting_export_summary': [] + }, "expenses_spent_at":[ { 'id': '1234', diff --git a/tests/test_fyle/test_queue.py b/tests/test_fyle/test_queue.py index 4e86f081..50ec07fb 100644 --- a/tests/test_fyle/test_queue.py +++ b/tests/test_fyle/test_queue.py @@ -1,5 +1,5 @@ from apps.fyle.models import Expense -from apps.workspaces.models import FyleCredential +from apps.workspaces.models import Workspace, FyleCredential from apps.netsuite.queue import __create_chain_and_run from apps.fyle.queue import async_post_accounting_export_summary, async_import_and_export_expenses @@ -37,4 +37,8 @@ def test_async_import_and_export_expenses(db): } } - async_import_and_export_expenses(body) + worksapce, _ = Workspace.objects.update_or_create( + fyle_org_id='or79Cob97KSh' + ) + + async_import_and_export_expenses(body, worksapce.id) diff --git a/tests/test_fyle/test_tasks.py b/tests/test_fyle/test_tasks.py index 84215640..01f0aad9 100644 --- a/tests/test_fyle/test_tasks.py +++ b/tests/test_fyle/test_tasks.py @@ -1,20 +1,25 @@ -from cmath import exp import pytest import json from django.db.models import Q from apps.fyle.models import ExpenseGroup, Expense, ExpenseGroupSettings from apps.tasks.models import TaskLog -from apps.fyle.tasks import create_expense_groups, schedule_expense_group_creation, post_accounting_export_summary +from apps.fyle.tasks import ( + create_expense_groups, + schedule_expense_group_creation, + post_accounting_export_summary, + update_non_exported_expenses +) from apps.workspaces.models import Configuration, FyleCredential, Workspace from .fixtures import data from django.urls import reverse +from rest_framework.exceptions import ValidationError +from rest_framework import status from tests.helper import dict_compare_keys from unittest import mock from apps.fyle.actions import mark_expenses_as_skipped from fyle.platform.exceptions import InvalidTokenError, InternalServerError - @pytest.mark.django_db() def test_create_expense_group(mocker, add_fyle_credentials): task_log, _ = TaskLog.objects.update_or_create( @@ -24,7 +29,7 @@ def test_create_expense_group(mocker, add_fyle_credentials): 'status': 'IN_PROGRESS' } ) - + expense_group_settings = ExpenseGroupSettings.objects.get(workspace_id=1) expense_group_settings.import_card_credits = True expense_group_settings.save() @@ -124,7 +129,6 @@ def test_create_expense_group_skipped_flow(mocker, api_client, add_fyle_credenti if expense.employee_email == 'jhonsnow@fyle.in': assert expense.is_skipped == True - @pytest.mark.django_db() def test_schedule_expense_group_creation(mocker, add_fyle_credentials): @@ -169,3 +173,56 @@ def test_post_accounting_export_summary(db, mocker): post_accounting_export_summary('or79Cob97KSh', 1) assert Expense.objects.filter(id=expense_id).first().accounting_export_summary['synced'] == True + + +def test_update_non_exported_expenses(db, create_temp_workspace, mocker, api_client): + expense = data['raw_expense'] + default_raw_expense = data['default_raw_expense'] + org_id = expense['org_id'] + payload = { + "resource": "EXPENSE", + "action": 'UPDATED_AFTER_APPROVAL', + "data": expense, + "reason": 'expense update testing', + } + + expense_created, _ = Expense.objects.update_or_create( + org_id=org_id, + expense_id='txhJLOSKs1iN', + workspace_id=1, + defaults=default_raw_expense + ) + expense_created.accounting_export_summary = {} + expense_created.save() + + workspace = Workspace.objects.filter(id=1).first() + workspace.fyle_org_id = org_id + workspace.save() + + assert expense_created.category == 'Old Category' + + update_non_exported_expenses(payload['data']) + + expense = Expense.objects.get(expense_id='txhJLOSKs1iN', org_id=org_id) + assert expense.category == 'ABN Withholding' + + expense.accounting_export_summary = {"synced": True, "state": "COMPLETE"} + expense.category = 'Old Category' + expense.save() + + update_non_exported_expenses(payload['data']) + expense = Expense.objects.get(expense_id='txhJLOSKs1iN', org_id=org_id) + assert expense.category == 'Old Category' + + try: + update_non_exported_expenses(payload['data']) + except ValidationError as e: + assert e.detail[0] == 'Workspace mismatch' + + url = reverse('exports', kwargs={'workspace_id': 1}) + response = api_client.post(url, data=payload, format='json') + assert response.status_code == status.HTTP_200_OK + + url = reverse('exports', kwargs={'workspace_id': 2}) + response = api_client.post(url, data=payload, format='json') + assert response.status_code == status.HTTP_400_BAD_REQUEST