From 0b487e56c3a75d135fc0d7700ef6a1d32ef546f4 Mon Sep 17 00:00:00 2001 From: Viswas Haridas <37623357+JustARatherRidiculouslyLongUsername@users.noreply.github.com> Date: Sat, 16 Nov 2024 00:18:07 +0530 Subject: [PATCH] test: add unit tests for split expense grouping and update existing fixtures (#401) * test: add unit tests for split expense grouping and update existing fixtures * refactor: lint * feat: add `split_expense_grouping` to the expense group settings serializer (#404) * feat: add `split_expense_grouping` to the expense group settings serializer * feat: implement split expense grouping functionality (#405) * feat: implement split expense grouping functionality * fix: skip only `expense_id` and `expense_number` while grouping while defining the fields to group split expenses by, skip only `expense_id` and `expense_number`, NOT `spent_at` or `posted_at` * feat: add script for split expense grouping (#406) Set the default split expense grouping config to `SINGLE_LINE_ITEM` for all old orgs --- apps/fyle/models.py | 54 ++++- .../apis/export_settings/serializers.py | 4 + ...lt-split-expense-grouping-for-old-orgs.sql | 5 + tests/test_fyle/conftest.py | 19 ++ tests/test_fyle/fixtures.py | 188 +++++++++++++++++- tests/test_fyle/test_models.py | 104 ++++++++++ tests/test_workspaces/fixtures.py | 1 + .../test_apis/test_clone_settings/fixtures.py | 2 + .../test_export_settings/fixtures.py | 2 + 9 files changed, 371 insertions(+), 8 deletions(-) create mode 100644 sql/scripts/027-default-split-expense-grouping-for-old-orgs.sql diff --git a/apps/fyle/models.py b/apps/fyle/models.py index c5269f09..8d4e5c9e 100644 --- a/apps/fyle/models.py +++ b/apps/fyle/models.py @@ -14,7 +14,7 @@ from fyle_accounting_mappings.models import ExpenseAttribute from apps.fyle.enums import ExpenseStateEnum, FundSourceEnum, PlatformExpensesEnum -from apps.workspaces.models import Workspace +from apps.workspaces.models import Workspace, WorkspaceGeneralSettings logger = logging.getLogger(__name__) logger.level = logging.INFO @@ -35,6 +35,7 @@ "spent_at", "expense_id", "posted_at", + "bank_transaction_id", ] @@ -515,13 +516,52 @@ def create_expense_groups_by_report_id_fund_source( corporate_credit_card_expenses = list( filter(lambda expense: expense.fund_source == "CCC", expense_objects) ) - corporate_credit_card_expense_groups = _group_expenses( - corporate_credit_card_expenses, - corporate_credit_card_expense_group_field, - workspace_id, - ) - expense_groups.extend(corporate_credit_card_expense_groups) + if corporate_credit_card_expenses: + workspace_general_settings = WorkspaceGeneralSettings.objects.get( + workspace_id=workspace_id + ) + ccc_export_module = workspace_general_settings.corporate_credit_card_expenses_object + + if ccc_export_module == "BANK TRANSACTION" and expense_group_settings.split_expense_grouping == 'MULTIPLE_LINE_ITEM': + ccc_expenses_without_bank_transaction_id = list( + filter(lambda expense: not expense.bank_transaction_id, corporate_credit_card_expenses) + ) + + ccc_expenses_with_bank_transaction_id = list( + filter(lambda expense: expense.bank_transaction_id, corporate_credit_card_expenses) + ) + + if ccc_expenses_without_bank_transaction_id: + groups_without_bank_transaction_id = _group_expenses( + ccc_expenses_without_bank_transaction_id, + corporate_credit_card_expense_group_field, + workspace_id, + ) + expense_groups.extend(groups_without_bank_transaction_id) + + if ccc_expenses_with_bank_transaction_id: + split_expense_group_fields = [ + field for field in corporate_credit_card_expense_group_field + if field not in ('expense_id', 'expense_number') + ] + split_expense_group_fields.append('bank_transaction_id') + + groups_with_bank_transaction_id = _group_expenses( + ccc_expenses_with_bank_transaction_id, + split_expense_group_fields, + workspace_id, + ) + expense_groups.extend(groups_with_bank_transaction_id) + + else: + corporate_credit_card_expense_groups = _group_expenses( + corporate_credit_card_expenses, + corporate_credit_card_expense_group_field, + workspace_id, + ) + + expense_groups.extend(corporate_credit_card_expense_groups) expense_group_objects = [] diff --git a/apps/workspaces/apis/export_settings/serializers.py b/apps/workspaces/apis/export_settings/serializers.py index 83a4d58b..8ebce381 100644 --- a/apps/workspaces/apis/export_settings/serializers.py +++ b/apps/workspaces/apis/export_settings/serializers.py @@ -64,6 +64,9 @@ class ExpenseGroupSettingsSerializer(serializers.ModelSerializer): ccc_expense_state = serializers.CharField( allow_null=True, allow_blank=True, required=False ) + split_expense_grouping = serializers.CharField( + allow_null=False, allow_blank=False, required=True + ) class Meta: model = ExpenseGroupSettings @@ -74,6 +77,7 @@ class Meta: "corporate_credit_card_expense_group_fields", "ccc_export_date_type", "ccc_expense_state", + "split_expense_grouping", ] diff --git a/sql/scripts/027-default-split-expense-grouping-for-old-orgs.sql b/sql/scripts/027-default-split-expense-grouping-for-old-orgs.sql new file mode 100644 index 00000000..89797023 --- /dev/null +++ b/sql/scripts/027-default-split-expense-grouping-for-old-orgs.sql @@ -0,0 +1,5 @@ +rollback; +begin; + +UPDATE expense_group_settings +SET split_expense_grouping = 'SINGLE_LINE_ITEM'; diff --git a/tests/test_fyle/conftest.py b/tests/test_fyle/conftest.py index 035000fc..47dc475e 100644 --- a/tests/test_fyle/conftest.py +++ b/tests/test_fyle/conftest.py @@ -32,3 +32,22 @@ def create_temp_workspace(db): reimbursable_export_date_type="current_date", ccc_export_date_type="spent_at", ) + + +@pytest.fixture +def update_config_for_split_expense_grouping(db): + def _update_config_for_split_expense_grouping(general_settings, expense_group_settings): + general_settings.corporate_credit_card_expenses_object = 'BANK TRANSACTION' + general_settings.save() + expense_group_settings.split_expense_grouping = 'SINGLE_LINE_ITEM' + expense_group_settings.corporate_credit_card_expense_group_fields = [ + 'expense_id', + 'claim_number', + 'fund_source', + 'employee_email', + 'report_id', + 'spent_at', + 'report_id' + ] + expense_group_settings.save() + return _update_config_for_split_expense_grouping diff --git a/tests/test_fyle/fixtures.py b/tests/test_fyle/fixtures.py index c26670cf..158e5e3c 100644 --- a/tests/test_fyle/fixtures.py +++ b/tests/test_fyle/fixtures.py @@ -236,7 +236,8 @@ 'tax_amount': None, 'tax_group_id': None, 'previous_export_state': None, - 'accounting_export_summary': [] + 'accounting_export_summary': [], + "bank_transaction_id": None, }, "expenses": [ { @@ -278,8 +279,189 @@ "Vehicle Type": "", "Fyle Categories": "", }, + "bank_transaction_id": None, }, ], + + "ccc_split_expenses": [ + { + "id": 15133, + "employee_email": "admin1@fyleforexport.updates", + "employee_name": "Theresa Brown", + "category": "Food", + "sub_category": None, + "project": "Project 8", + "org_id": "oryoUx39Vz66", + "expense_number": "E/2024/11/T/7", + "claim_number": "C/2024/11/R/3", + "amount": 21.208, + "currency": "USD", + "foreign_amount": None, + "foreign_currency": None, + "settlement_id": None, + "reimbursable": False, + "state": "APPROVED", + "vendor": "Dominos Pizza", + "cost_center": None, + "corporate_card_id": "baccZ5Ww7s9mLu", + "purpose": None, + "report_id": "rprwO90Ul12E", + "billable": False, + "file_ids": [], + "spent_at": '2024-11-04T00:00:00Z', + "approved_at": '2024-11-04T15:45:02.657341Z', + "posted_at": None, + "expense_created_at": '2024-11-04T15:44:46.752178Z', + "expense_updated_at": '2024-11-04T15:45:02.487787Z', + "created_at": '2024-11-07T20:20:05.416618Z', + "updated_at": '2024-11-07T20:20:05.416629Z', + "fund_source": "CCC", + "verified_at": None, + "custom_properties": {"Custom Expense Field": None}, + "paid_on_xero": False, + "paid_on_fyle": False, + "tax_amount": 0.0, + "tax_group_id": None, + "accounting_export_summary": {}, + "previous_export_state": None, + "workspace_id": 399, + "source_account_type": "PERSONAL_CORPORATE_CREDIT_CARD_ACCOUNT", + "bank_transaction_id": "btxnQe02cfpD9n" + }, + { + "id": 15134, + "employee_email": "admin1@fyleforexport.updates", + "employee_name": "Theresa Brown", + "category": "Food", + "sub_category": None, + "project": "Project 10", + "org_id": "oryoUx39Vz66", + "expense_number": "E/2024/11/T/6", + "claim_number": "C/2024/11/R/3", + "amount": 31.812, + "currency": "USD", + "foreign_amount": None, + "foreign_currency": None, + "settlement_id": None, + "reimbursable": False, + "state": "APPROVED", + "vendor": "Dominos Pizza", + "cost_center": None, + "corporate_card_id": "baccZ5Ww7s9mLu", + "purpose": None, + "report_id": "rprwO90Ul12E", + "billable": False, + "file_ids": [], + "spent_at": '2024-11-04T00:00:00Z', + "approved_at": '2024-11-04T15:45:02.657341Z', + "posted_at": None, + "expense_created_at": '2024-11-04T15:44:46.581967Z', + "expense_updated_at": '2024-11-04T15:45:02.283785Z', + "created_at": '2024-11-07T20:20:05.438423Z', + "updated_at": '2024-11-07T20:20:05.438431Z', + "fund_source": "CCC", + "verified_at": None, + "custom_properties": {"Custom Expense Field": None}, + "paid_on_xero": False, + "paid_on_fyle": False, + "tax_amount": 0.0, + "tax_group_id": None, + "accounting_export_summary": {}, + "previous_export_state": None, + "workspace_id": 399, + "source_account_type": "PERSONAL_CORPORATE_CREDIT_CARD_ACCOUNT", + "bank_transaction_id": "btxnQe02cfpD9n" + }, + { + "id": 15135, + "employee_email": "admin1@fyleforexport.updates", + "employee_name": "Theresa Brown", + "category": "Food", + "sub_category": None, + "project": "Project 10", + "org_id": "oryoUx39Vz66", + "expense_number": "E/2024/11/T/8", + "claim_number": "C/2024/11/R/3", + "amount": 23.492, + "currency": "USD", + "foreign_amount": None, + "foreign_currency": None, + "settlement_id": None, + "reimbursable": False, + "state": "APPROVED", + "vendor": "Dominos Pizza", + "cost_center": None, + "corporate_card_id": "baccZ5Ww7s9mLu", + "purpose": None, + "report_id": "rprwO90Ul12E", + "billable": False, + "file_ids": [], + "spent_at": '2024-11-04T00:00:00Z', + "approved_at": '2024-11-04T15:45:02.657341Z', + "posted_at": None, + "expense_created_at": '2024-11-04T15:44:46.581967Z', + "expense_updated_at": '2024-11-04T15:45:02.283785Z', + "created_at": '2024-11-07T20:20:05.438423Z', + "updated_at": '2024-11-07T20:20:05.438431Z', + "fund_source": "CCC", + "verified_at": None, + "custom_properties": {"Custom Expense Field": None}, + "paid_on_xero": False, + "paid_on_fyle": False, + "tax_amount": 0.0, + "tax_group_id": None, + "accounting_export_summary": {}, + "previous_export_state": None, + "workspace_id": 399, + "source_account_type": "PERSONAL_CORPORATE_CREDIT_CARD_ACCOUNT", + "bank_transaction_id": "btxnQe02cfpD9n" + }, + { + "id": 15136, + "employee_email": "admin1@fyleforexport.updates", + "employee_name": "Theresa Brown", + "category": "Food", + "sub_category": None, + "project": "Project 10", + "org_id": "oryoUx39Vz66", + "expense_number": "E/2024/11/T/9", + "claim_number": "C/2024/11/R/3", + "amount": 26.722, + "currency": "USD", + "foreign_amount": None, + "foreign_currency": None, + "settlement_id": None, + "reimbursable": False, + "state": "APPROVED", + "vendor": "Dominos Pizza", + "cost_center": None, + "corporate_card_id": "baccZ5Ww7s9mLu", + "purpose": None, + "report_id": "rprwO90Ul12E", + "billable": False, + "file_ids": [], + "spent_at": '2024-11-04T00:00:00Z', + "approved_at": '2024-11-04T15:45:02.657341Z', + "posted_at": None, + "expense_created_at": '2024-11-04T15:44:46.581967Z', + "expense_updated_at": '2024-11-04T15:45:02.283785Z', + "created_at": '2024-11-07T20:20:05.438423Z', + "updated_at": '2024-11-07T20:20:05.438431Z', + "fund_source": "CCC", + "verified_at": None, + "custom_properties": {"Custom Expense Field": None}, + "paid_on_xero": False, + "paid_on_fyle": False, + "tax_amount": 0.0, + "tax_group_id": None, + "accounting_export_summary": {}, + "previous_export_state": None, + "workspace_id": 399, + "source_account_type": "PERSONAL_CORPORATE_CREDIT_CARD_ACCOUNT", + "bank_transaction_id": "btxnQe02cfpD9n" + }, + ], + "eliminated_expenses": [ { "id": "tx6wOnBVaumk", @@ -320,6 +502,7 @@ "Vehicle Type": "", "Fyle Categories": "", }, + "bank_transaction_id": None, } ], "expense_group_id": { @@ -366,6 +549,7 @@ "import_card_credits": "false", "created_at": "2021-11-15T08:46:16.069944Z", "updated_at": "2021-11-15T08:46:16.069986Z", + "split_expense_grouping": "MULTIPLE_LINE_ITEM", "workspace": 1, }, "reimbursements": [ @@ -396,6 +580,7 @@ "ccc_expense_state": "PAYMENT_PROCESSING", "reimbursable_export_date_type": "spent_at", "ccc_export_date_type": "spent_at", + "split_expense_grouping": "MULTIPLE_LINE_ITEM", }, "expense_fields_response": [ {"attribute_type": "COST_CENTER", "display_name": "Cost Center"}, @@ -594,6 +779,7 @@ "paid_on_xero": False, "tax_amount": "None", "tax_group_id": "None", + "bank_transaction_id": None, } ], "fund_source": "PERSONAL", diff --git a/tests/test_fyle/test_models.py b/tests/test_fyle/test_models.py index ea7e40e1..dfb0a694 100644 --- a/tests/test_fyle/test_models.py +++ b/tests/test_fyle/test_models.py @@ -7,6 +7,7 @@ get_default_expense_group_fields, get_default_expense_state, ) +from apps.workspaces.models import WorkspaceGeneralSettings from tests.test_fyle.fixtures import data @@ -104,3 +105,106 @@ def test_create_expense_groups_by_report_id_fund_source(db): expense_groups = ExpenseGroup.objects.last() assert expense_groups.exported_at == None + + +def test_split_expense_grouping_with_no_bank_transaction_id(db, update_config_for_split_expense_grouping): + ''' + Test for grouping of 2 expenses with no bank transaction id + ''' + workspace_id = 1 + + # Update settings + general_settings = WorkspaceGeneralSettings.objects.get(workspace_id=workspace_id) + expense_group_settings = ExpenseGroupSettings.objects.get(workspace_id=workspace_id) + update_config_for_split_expense_grouping(general_settings, expense_group_settings) + + # Get reference to expense objects + expenses = data['ccc_split_expenses'][:2] + for expense in expenses: + expense['bank_transaction_id'] = None + + Expense.create_expense_objects(expenses, workspace_id=workspace_id) + expense_objects = Expense.objects.filter(expense_id__in=[expense['id'] for expense in expenses]) + + assert len(expense_objects) == 2, f'Expected 2 expenses, got {len(expense_objects)}' + + # Test for SINGLE_LINE_ITEM split expense grouping + groups = ExpenseGroup.create_expense_groups_by_report_id_fund_source(expense_objects, workspace_id) + assert len(groups) == 2, f'Expected 2 groups, got {len(groups)}' + + # Test for MULTIPLE_LINE_ITEM split expense grouping + expense_group_settings.split_expense_grouping = 'MULTIPLE_LINE_ITEM' + expense_group_settings.save() + + groups = ExpenseGroup.create_expense_groups_by_report_id_fund_source(expense_objects, workspace_id) + assert len(groups) == 2, f'Expected 2 groups, got {len(groups)}' + + +def test_split_expense_grouping_with_same_and_different_ids(db, update_config_for_split_expense_grouping): + ''' + Test for grouping of 2 expenses with the same bank transaction id, + and one expense with a different bank transaction id + ''' + workspace_id = 1 + + # Update settings + general_settings = WorkspaceGeneralSettings.objects.get(workspace_id=workspace_id) + expense_group_settings = ExpenseGroupSettings.objects.get(workspace_id=workspace_id) + update_config_for_split_expense_grouping(general_settings, expense_group_settings) + + # Get reference to expense objects + expenses = data['ccc_split_expenses'][:3] + expenses[0]['bank_transaction_id'] = 'sample_1' + expenses[1]['bank_transaction_id'] = 'sample_1' + expenses[2]['bank_transaction_id'] = 'sample_2' + + Expense.create_expense_objects(expenses, workspace_id=workspace_id) + expense_objects = Expense.objects.filter(expense_id__in=[expense['id'] for expense in expenses]) + + assert len(expense_objects) == 3, f'Expected 3 expenses, got {len(expense_objects)}' + + # Test for SINGLE_LINE_ITEM split expense grouping + groups = ExpenseGroup.create_expense_groups_by_report_id_fund_source(expense_objects, workspace_id) + assert len(groups) == 3, f'Expected 3 groups, got {len(groups)}' + + # Test for MULTIPLE_LINE_ITEM split expense grouping + expense_group_settings.split_expense_grouping = 'MULTIPLE_LINE_ITEM' + expense_group_settings.save() + + groups = ExpenseGroup.create_expense_groups_by_report_id_fund_source(expense_objects, workspace_id) + assert len(groups) == 2, f'Expected 2 groups, got {len(groups)}' + + +def test_split_expense_grouping_pairs_of_same_ids(db, update_config_for_split_expense_grouping): + ''' + Test for grouping of 2 pairs of expenses with the same bank transaction ids + ''' + workspace_id = 1 + + # Update settings + general_settings = WorkspaceGeneralSettings.objects.get(workspace_id=workspace_id) + expense_group_settings = ExpenseGroupSettings.objects.get(workspace_id=workspace_id) + update_config_for_split_expense_grouping(general_settings, expense_group_settings) + + # Get reference to expense objects + expenses = data['ccc_split_expenses'][:4] + expenses[0]['bank_transaction_id'] = 'sample_1' + expenses[1]['bank_transaction_id'] = 'sample_1' + expenses[2]['bank_transaction_id'] = 'sample_2' + expenses[3]['bank_transaction_id'] = 'sample_2' + + Expense.create_expense_objects(expenses, workspace_id=workspace_id) + expense_objects = Expense.objects.filter(expense_id__in=[expense['id'] for expense in expenses]) + + assert len(expense_objects) == 4, f'Expected 4 expenses, got {len(expense_objects)}' + + # Test for SINGLE_LINE_ITEM split expense grouping + groups = ExpenseGroup.create_expense_groups_by_report_id_fund_source(expense_objects, workspace_id) + assert len(groups) == 4, f'Expected 4 groups, got {len(groups)}' + + # Test for MULTIPLE_LINE_ITEM split expense grouping + expense_group_settings.split_expense_grouping = 'MULTIPLE_LINE_ITEM' + expense_group_settings.save() + + groups = ExpenseGroup.create_expense_groups_by_report_id_fund_source(expense_objects, workspace_id) + assert len(groups) == 2, f'Expected 2 groups, got {len(groups)}' diff --git a/tests/test_workspaces/fixtures.py b/tests/test_workspaces/fixtures.py index b8ae01ed..3d58f13e 100644 --- a/tests/test_workspaces/fixtures.py +++ b/tests/test_workspaces/fixtures.py @@ -96,6 +96,7 @@ }, "paid_on_qbo": False, "payment_number": "P/2022/05/R/7", + "bank_transaction_id": None, } ], "workspace_schedule": { diff --git a/tests/test_workspaces/test_apis/test_clone_settings/fixtures.py b/tests/test_workspaces/test_apis/test_clone_settings/fixtures.py index 2add1b98..d24a8009 100644 --- a/tests/test_workspaces/test_apis/test_clone_settings/fixtures.py +++ b/tests/test_workspaces/test_apis/test_clone_settings/fixtures.py @@ -27,6 +27,7 @@ "ccc_export_date_type": "spent_at", "ccc_expense_state": "PAYMENT_PROCESSING", "import_card_credits": True, + "split_expense_grouping": "MULTIPLE_LINE_ITEM", }, "general_mappings": { "bank_account": {"id": "10", "name": "Visa"}, @@ -95,6 +96,7 @@ "reimbursable_export_date_type": "current_date", "ccc_expense_state": "PAID", "ccc_export_date_type": "spent_at", + "split_expense_grouping": "MULTIPLE_LINE_ITEM", }, "workspace_general_settings": { "reimbursable_expenses_object": "PURCHASE BILL", diff --git a/tests/test_workspaces/test_apis/test_export_settings/fixtures.py b/tests/test_workspaces/test_apis/test_export_settings/fixtures.py index df45bd54..2d9d2aba 100644 --- a/tests/test_workspaces/test_apis/test_export_settings/fixtures.py +++ b/tests/test_workspaces/test_apis/test_export_settings/fixtures.py @@ -7,6 +7,7 @@ "import_card_credits": True, "ccc_export_date_type": "", "expense_state": "", + "split_expense_grouping": "MULTIPLE_LINE_ITEM", }, "workspace_general_settings": { "reimbursable_expenses_object": "PURCHASE BILL", @@ -45,6 +46,7 @@ "ccc_export_date_type": "spent_at", "ccc_expense_state": "PAYMENT_PROCESSING", "import_card_credits": True, + "split_expense_grouping": "MULTIPLE_LINE_ITEM", }, "general_mappings": { "bank_account": {"id": "10", "name": "Visa"},