test: add unit tests for split expense grouping and update existing f…

…ixtures (#401)

* test: add unit tests for split expense grouping and update existing fixtures

* refactor: lint

* feat: add `split_expense_grouping` to the expense group settings serializer (#404)

* feat: add `split_expense_grouping` to the expense group settings serializer

* feat: implement split expense grouping functionality (#405)

* feat: implement split expense grouping functionality

* fix: skip only `expense_id` and `expense_number` while grouping

while defining the fields to group split expenses by, skip only `expense_id` and `expense_number`, NOT `spent_at` or `posted_at`

* feat: add script for split expense grouping (#406)

Set the default split expense grouping config to `SINGLE_LINE_ITEM` for all old orgs

apps/fyle/models.py

@@ -14,7 +14,7 @@
 from fyle_accounting_mappings.models import ExpenseAttribute
 
 from apps.fyle.enums import ExpenseStateEnum, FundSourceEnum, PlatformExpensesEnum
-from apps.workspaces.models import Workspace
+from apps.workspaces.models import Workspace, WorkspaceGeneralSettings
 
 logger = logging.getLogger(__name__)
 logger.level = logging.INFO
@@ -35,6 +35,7 @@
     "spent_at",
     "expense_id",
     "posted_at",
+    "bank_transaction_id",
 ]
 
 
@@ -515,13 +516,52 @@ def create_expense_groups_by_report_id_fund_source(
         corporate_credit_card_expenses = list(
             filter(lambda expense: expense.fund_source == "CCC", expense_objects)
         )
-        corporate_credit_card_expense_groups = _group_expenses(
-            corporate_credit_card_expenses,
-            corporate_credit_card_expense_group_field,
-            workspace_id,
-        )
 
-        expense_groups.extend(corporate_credit_card_expense_groups)
+        if corporate_credit_card_expenses:
+            workspace_general_settings = WorkspaceGeneralSettings.objects.get(
+                workspace_id=workspace_id
+            )
+            ccc_export_module = workspace_general_settings.corporate_credit_card_expenses_object
+
+            if ccc_export_module == "BANK TRANSACTION" and expense_group_settings.split_expense_grouping == 'MULTIPLE_LINE_ITEM':
+                ccc_expenses_without_bank_transaction_id = list(
+                    filter(lambda expense: not expense.bank_transaction_id, corporate_credit_card_expenses)
+                )
+
+                ccc_expenses_with_bank_transaction_id = list(
+                    filter(lambda expense: expense.bank_transaction_id, corporate_credit_card_expenses)
+                )
+
+                if ccc_expenses_without_bank_transaction_id:
+                    groups_without_bank_transaction_id = _group_expenses(
+                        ccc_expenses_without_bank_transaction_id,
+                        corporate_credit_card_expense_group_field,
+                        workspace_id,
+                    )
+                    expense_groups.extend(groups_without_bank_transaction_id)
+
+                if ccc_expenses_with_bank_transaction_id:
+                    split_expense_group_fields = [
+                        field for field in corporate_credit_card_expense_group_field
+                        if field not in ('expense_id', 'expense_number')
+                    ]
+                    split_expense_group_fields.append('bank_transaction_id')
+
+                    groups_with_bank_transaction_id = _group_expenses(
+                        ccc_expenses_with_bank_transaction_id,
+                        split_expense_group_fields,
+                        workspace_id,
+                    )
+                    expense_groups.extend(groups_with_bank_transaction_id)
+
+            else:
+                corporate_credit_card_expense_groups = _group_expenses(
+                    corporate_credit_card_expenses,
+                    corporate_credit_card_expense_group_field,
+                    workspace_id,
+                )
+
+                expense_groups.extend(corporate_credit_card_expense_groups)
 
         expense_group_objects = []
 

apps/workspaces/apis/export_settings/serializers.py

@@ -64,6 +64,9 @@ class ExpenseGroupSettingsSerializer(serializers.ModelSerializer):
     ccc_expense_state = serializers.CharField(
         allow_null=True, allow_blank=True, required=False
     )
+    split_expense_grouping = serializers.CharField(
+        allow_null=False, allow_blank=False, required=True
+    )
 
     class Meta:
         model = ExpenseGroupSettings
@@ -74,6 +77,7 @@ class Meta:
             "corporate_credit_card_expense_group_fields",
             "ccc_export_date_type",
             "ccc_expense_state",
+            "split_expense_grouping",
         ]
 
 

sql/scripts/027-default-split-expense-grouping-for-old-orgs.sql

@@ -0,0 +1,5 @@
+rollback;
+begin;
+
+UPDATE expense_group_settings
+SET split_expense_grouping = 'SINGLE_LINE_ITEM';

tests/test_fyle/conftest.py

@@ -32,3 +32,22 @@ def create_temp_workspace(db):
         reimbursable_export_date_type="current_date",
         ccc_export_date_type="spent_at",
     )
+
+
+@pytest.fixture
+def update_config_for_split_expense_grouping(db):
+    def _update_config_for_split_expense_grouping(general_settings, expense_group_settings):
+        general_settings.corporate_credit_card_expenses_object = 'BANK TRANSACTION'
+        general_settings.save()
+        expense_group_settings.split_expense_grouping = 'SINGLE_LINE_ITEM'
+        expense_group_settings.corporate_credit_card_expense_group_fields = [
+            'expense_id',
+            'claim_number',
+            'fund_source',
+            'employee_email',
+            'report_id',
+            'spent_at',
+            'report_id'
+        ]
+        expense_group_settings.save()
+    return _update_config_for_split_expense_grouping

tests/test_fyle/fixtures.py

@@ -236,7 +236,8 @@
         'tax_amount': None,
         'tax_group_id': None,
         'previous_export_state': None,
-        'accounting_export_summary': []
+        'accounting_export_summary': [],
+        "bank_transaction_id": None,
     },
     "expenses": [
         {
@@ -278,8 +279,189 @@
                 "Vehicle Type": "",
                 "Fyle Categories": "",
             },
+            "bank_transaction_id": None,
         },
     ],
+
+    "ccc_split_expenses": [
+        {
+            "id": 15133,
+            "employee_email": "[email protected]",
+            "employee_name": "Theresa Brown",
+            "category": "Food",
+            "sub_category": None,
+            "project": "Project 8",
+            "org_id": "oryoUx39Vz66",
+            "expense_number": "E/2024/11/T/7",
+            "claim_number": "C/2024/11/R/3",
+            "amount": 21.208,
+            "currency": "USD",
+            "foreign_amount": None,
+            "foreign_currency": None,
+            "settlement_id": None,
+            "reimbursable": False,
+            "state": "APPROVED",
+            "vendor": "Dominos Pizza",
+            "cost_center": None,
+            "corporate_card_id": "baccZ5Ww7s9mLu",
+            "purpose": None,
+            "report_id": "rprwO90Ul12E",
+            "billable": False,
+            "file_ids": [],
+            "spent_at": '2024-11-04T00:00:00Z',
+            "approved_at": '2024-11-04T15:45:02.657341Z',
+            "posted_at": None,
+            "expense_created_at": '2024-11-04T15:44:46.752178Z',
+            "expense_updated_at": '2024-11-04T15:45:02.487787Z',
+            "created_at": '2024-11-07T20:20:05.416618Z',
+            "updated_at": '2024-11-07T20:20:05.416629Z',
+            "fund_source": "CCC",
+            "verified_at": None,
+            "custom_properties": {"Custom Expense Field": None},
+            "paid_on_xero": False,
+            "paid_on_fyle": False,
+            "tax_amount": 0.0,
+            "tax_group_id": None,
+            "accounting_export_summary": {},
+            "previous_export_state": None,
+            "workspace_id": 399,
+            "source_account_type": "PERSONAL_CORPORATE_CREDIT_CARD_ACCOUNT",
+            "bank_transaction_id": "btxnQe02cfpD9n"
+        },
+        {
+            "id": 15134,
+            "employee_email": "[email protected]",
+            "employee_name": "Theresa Brown",
+            "category": "Food",
+            "sub_category": None,
+            "project": "Project 10",
+            "org_id": "oryoUx39Vz66",
+            "expense_number": "E/2024/11/T/6",
+            "claim_number": "C/2024/11/R/3",
+            "amount": 31.812,
+            "currency": "USD",
+            "foreign_amount": None,
+            "foreign_currency": None,
+            "settlement_id": None,
+            "reimbursable": False,
+            "state": "APPROVED",
+            "vendor": "Dominos Pizza",
+            "cost_center": None,
+            "corporate_card_id": "baccZ5Ww7s9mLu",
+            "purpose": None,
+            "report_id": "rprwO90Ul12E",
+            "billable": False,
+            "file_ids": [],
+            "spent_at": '2024-11-04T00:00:00Z',
+            "approved_at": '2024-11-04T15:45:02.657341Z',
+            "posted_at": None,
+            "expense_created_at": '2024-11-04T15:44:46.581967Z',
+            "expense_updated_at": '2024-11-04T15:45:02.283785Z',
+            "created_at": '2024-11-07T20:20:05.438423Z',
+            "updated_at": '2024-11-07T20:20:05.438431Z',
+            "fund_source": "CCC",
+            "verified_at": None,
+            "custom_properties": {"Custom Expense Field": None},
+            "paid_on_xero": False,
+            "paid_on_fyle": False,
+            "tax_amount": 0.0,
+            "tax_group_id": None,
+            "accounting_export_summary": {},
+            "previous_export_state": None,
+            "workspace_id": 399,
+            "source_account_type": "PERSONAL_CORPORATE_CREDIT_CARD_ACCOUNT",
+            "bank_transaction_id": "btxnQe02cfpD9n"
+        },
+        {
+            "id": 15135,
+            "employee_email": "[email protected]",
+            "employee_name": "Theresa Brown",
+            "category": "Food",
+            "sub_category": None,
+            "project": "Project 10",
+            "org_id": "oryoUx39Vz66",
+            "expense_number": "E/2024/11/T/8",
+            "claim_number": "C/2024/11/R/3",
+            "amount": 23.492,
+            "currency": "USD",
+            "foreign_amount": None,
+            "foreign_currency": None,
+            "settlement_id": None,
+            "reimbursable": False,
+            "state": "APPROVED",
+            "vendor": "Dominos Pizza",
+            "cost_center": None,
+            "corporate_card_id": "baccZ5Ww7s9mLu",
+            "purpose": None,
+            "report_id": "rprwO90Ul12E",
+            "billable": False,
+            "file_ids": [],
+            "spent_at": '2024-11-04T00:00:00Z',
+            "approved_at": '2024-11-04T15:45:02.657341Z',
+            "posted_at": None,
+            "expense_created_at": '2024-11-04T15:44:46.581967Z',
+            "expense_updated_at": '2024-11-04T15:45:02.283785Z',
+            "created_at": '2024-11-07T20:20:05.438423Z',
+            "updated_at": '2024-11-07T20:20:05.438431Z',
+            "fund_source": "CCC",
+            "verified_at": None,
+            "custom_properties": {"Custom Expense Field": None},
+            "paid_on_xero": False,
+            "paid_on_fyle": False,
+            "tax_amount": 0.0,
+            "tax_group_id": None,
+            "accounting_export_summary": {},
+            "previous_export_state": None,
+            "workspace_id": 399,
+            "source_account_type": "PERSONAL_CORPORATE_CREDIT_CARD_ACCOUNT",
+            "bank_transaction_id": "btxnQe02cfpD9n"
+        },
+        {
+            "id": 15136,
+            "employee_email": "[email protected]",
+            "employee_name": "Theresa Brown",
+            "category": "Food",
+            "sub_category": None,
+            "project": "Project 10",
+            "org_id": "oryoUx39Vz66",
+            "expense_number": "E/2024/11/T/9",
+            "claim_number": "C/2024/11/R/3",
+            "amount": 26.722,
+            "currency": "USD",
+            "foreign_amount": None,
+            "foreign_currency": None,
+            "settlement_id": None,
+            "reimbursable": False,
+            "state": "APPROVED",
+            "vendor": "Dominos Pizza",
+            "cost_center": None,
+            "corporate_card_id": "baccZ5Ww7s9mLu",
+            "purpose": None,
+            "report_id": "rprwO90Ul12E",
+            "billable": False,
+            "file_ids": [],
+            "spent_at": '2024-11-04T00:00:00Z',
+            "approved_at": '2024-11-04T15:45:02.657341Z',
+            "posted_at": None,
+            "expense_created_at": '2024-11-04T15:44:46.581967Z',
+            "expense_updated_at": '2024-11-04T15:45:02.283785Z',
+            "created_at": '2024-11-07T20:20:05.438423Z',
+            "updated_at": '2024-11-07T20:20:05.438431Z',
+            "fund_source": "CCC",
+            "verified_at": None,
+            "custom_properties": {"Custom Expense Field": None},
+            "paid_on_xero": False,
+            "paid_on_fyle": False,
+            "tax_amount": 0.0,
+            "tax_group_id": None,
+            "accounting_export_summary": {},
+            "previous_export_state": None,
+            "workspace_id": 399,
+            "source_account_type": "PERSONAL_CORPORATE_CREDIT_CARD_ACCOUNT",
+            "bank_transaction_id": "btxnQe02cfpD9n"
+        },
+    ],
+
     "eliminated_expenses": [
         {
             "id": "tx6wOnBVaumk",
@@ -320,6 +502,7 @@
                 "Vehicle Type": "",
                 "Fyle Categories": "",
             },
+            "bank_transaction_id": None,
         }
     ],
     "expense_group_id": {
@@ -366,6 +549,7 @@
         "import_card_credits": "false",
         "created_at": "2021-11-15T08:46:16.069944Z",
         "updated_at": "2021-11-15T08:46:16.069986Z",
+        "split_expense_grouping": "MULTIPLE_LINE_ITEM",
         "workspace": 1,
     },
     "reimbursements": [
@@ -396,6 +580,7 @@
         "ccc_expense_state": "PAYMENT_PROCESSING",
         "reimbursable_export_date_type": "spent_at",
         "ccc_export_date_type": "spent_at",
+        "split_expense_grouping": "MULTIPLE_LINE_ITEM",
     },
     "expense_fields_response": [
         {"attribute_type": "COST_CENTER", "display_name": "Cost Center"},
@@ -594,6 +779,7 @@
                 "paid_on_xero": False,
                 "tax_amount": "None",
                 "tax_group_id": "None",
+                "bank_transaction_id": None,
             }
         ],
         "fund_source": "PERSONAL",