diff --git a/apps/fyle/migrations/0027_expensegroup_employee_name.py b/apps/fyle/migrations/0027_expensegroup_employee_name.py new file mode 100644 index 00000000..58422342 --- /dev/null +++ b/apps/fyle/migrations/0027_expensegroup_employee_name.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.14 on 2023-11-29 11:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('fyle', '0026_auto_20231025_0913'), + ] + + operations = [ + migrations.AddField( + model_name='expensegroup', + name='employee_name', + field=models.CharField(help_text='Expense Group Employee Name', max_length=100, null=True), + ), + ] diff --git a/apps/fyle/migrations/0028_expensegroup_export_url.py b/apps/fyle/migrations/0028_expensegroup_export_url.py new file mode 100644 index 00000000..0bcdbbab --- /dev/null +++ b/apps/fyle/migrations/0028_expensegroup_export_url.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.14 on 2023-11-22 10:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('fyle', '0027_expensegroup_employee_name'), + ] + + operations = [ + migrations.AddField( + model_name='expensegroup', + name='export_url', + field=models.CharField(help_text='Netsuite URL for the exported expenses', max_length=255, null=True), + ), + ] diff --git a/apps/fyle/models.py b/apps/fyle/models.py index fed23cc2..9fcf76bb 100644 --- a/apps/fyle/models.py +++ b/apps/fyle/models.py @@ -325,8 +325,10 @@ class ExpenseGroup(models.Model): help_text='To which workspace this expense group belongs to') fund_source = models.CharField(max_length=255, help_text='Expense fund source') expenses = models.ManyToManyField(Expense, help_text="Expenses under this Expense Group") + employee_name = models.CharField(max_length=100, help_text='Expense Group Employee Name', null=True) description = JSONField(max_length=255, help_text='Description', null=True) response_logs = JSONField(help_text='Reponse log of the export', null=True) + export_url = models.CharField(max_length=255, help_text='Netsuite URL for the exported expenses', null=True) created_at = models.DateTimeField(auto_now_add=True, help_text='Created at') exported_at = models.DateTimeField(help_text='Exported at', null=True) updated_at = models.DateTimeField(auto_now=True, help_text='Updated at') @@ -392,7 +394,14 @@ def create_expense_groups_by_report_id_fund_source(expense_objects: List[Expense if expense_group_settings.ccc_export_date_type == 'last_spent_at': expense_group['last_spent_at'] = Expense.objects.filter( id__in=expense_group['expense_ids']).order_by('-spent_at').first().spent_at - + + employee_name = Expense.objects.filter( + id__in=expense_group['expense_ids'] + ).first().employee_name + + employee_name = Expense.objects.filter( + id__in=expense_group['expense_ids'] + ).first().employee_name expense_ids = expense_group['expense_ids'] expense_group.pop('total') expense_group.pop('expense_ids') @@ -407,7 +416,8 @@ def create_expense_groups_by_report_id_fund_source(expense_objects: List[Expense expense_group_object = ExpenseGroup.objects.create( workspace_id=workspace_id, fund_source=expense_group['fund_source'], - description=expense_group + description=expense_group, + employee_name=employee_name ) expense_group_object.expenses.add(*expense_ids) diff --git a/apps/fyle/serializers.py b/apps/fyle/serializers.py index 0cc5d045..71d608e1 100644 --- a/apps/fyle/serializers.py +++ b/apps/fyle/serializers.py @@ -5,13 +5,24 @@ from .models import Expense, ExpenseFilter, ExpenseGroup, ExpenseGroupSettings +class ExpenseSerializer(serializers.ModelSerializer): + """ + Expense serializer + """ + class Meta: + model = Expense + fields = ['updated_at', 'claim_number', 'employee_email', 'employee_name', 'fund_source', 'expense_number', 'vendor', 'category', 'amount', + 'report_id', 'settlement_id', 'expense_id'] + class ExpenseGroupSerializer(serializers.ModelSerializer): """ Expense group serializer """ + expenses = ExpenseSerializer(many=True) class Meta: model = ExpenseGroup fields = '__all__' + extra_fields = ['expenses'] class ExpenseGroupExpenseSerializer(serializers.ModelSerializer): @@ -60,12 +71,3 @@ def create(self, validated_data): ) return expense_filter - - -class ExpenseSerializer(serializers.ModelSerializer): - """ - Expense serializer - """ - class Meta: - model = Expense - fields = ['updated_at', 'claim_number', 'employee_email', 'employee_name', 'fund_source'] diff --git a/apps/fyle/tasks.py b/apps/fyle/tasks.py index f3ed39f8..e3b9ab40 100644 --- a/apps/fyle/tasks.py +++ b/apps/fyle/tasks.py @@ -24,27 +24,32 @@ 'CCC': 'PERSONAL_CORPORATE_CREDIT_CARD_ACCOUNT' } -def schedule_expense_group_creation(workspace_id: int): - """ - Schedule Expense group creation - :param workspace_id: Workspace id - :param user: User email - :return: None - """ +def get_task_log_and_fund_source(workspace_id: int): task_log, _ = TaskLog.objects.update_or_create( workspace_id=workspace_id, type='FETCHING_EXPENSES', defaults={ - 'status': 'IN_PROGRESS' + 'status': 'IN_PROGRESS' } ) configuration = Configuration.objects.get(workspace_id=workspace_id) - fund_source = ['PERSONAL'] if configuration.corporate_credit_card_expenses_object is not None: fund_source.append('CCC') + return task_log, fund_source, configuration + +def schedule_expense_group_creation(workspace_id: int): + """ + Schedule Expense group creation + :param workspace_id: Workspace id + :param user: User email + :return: None + """ + + task_log, fund_source, configuration = get_task_log_and_fund_source(workspace_id) + async_task('apps.fyle.tasks.create_expense_groups', workspace_id, configuration, fund_source, task_log) diff --git a/apps/fyle/urls.py b/apps/fyle/urls.py index ceee784b..2333ba8c 100644 --- a/apps/fyle/urls.py +++ b/apps/fyle/urls.py @@ -2,7 +2,7 @@ from django.urls import path -from .views import ExpenseGroupView, ExpenseGroupByIdView, ExpenseGroupScheduleView, ExportableExpenseGroupsView, FyleFieldsView, ExpenseView,\ +from .views import ExpenseGroupSyncView, ExpenseGroupView, ExpenseGroupByIdView, ExpenseGroupScheduleView, ExportableExpenseGroupsView, FyleFieldsView, ExpenseView,\ ExpenseAttributesView, ExpenseGroupSettingsView, SyncFyleDimensionView, RefreshFyleDimensionView,\ ExpenseGroupCountView, ExpenseFilterView, ExpenseGroupExpenseView, CustomFieldView @@ -13,7 +13,8 @@ path('expense_groups//', ExpenseGroupByIdView.as_view(), name='expense-group-by-id'), path('expense_groups//expenses/', ExpenseGroupExpenseView.as_view(), name='expense-group-expenses'), path('expense_group_settings/', ExpenseGroupSettingsView.as_view(), name='expense-group-settings'), - path('exportable_expense_groups/', ExportableExpenseGroupsView.as_view(), name='expense-expense-groups') + path('exportable_expense_groups/', ExportableExpenseGroupsView.as_view(), name='expense-expense-groups'), + path('expense_groups/sync/', ExpenseGroupSyncView.as_view(), name='sync-expense-groups'), ] fyle_dimension_paths = [ diff --git a/apps/fyle/views.py b/apps/fyle/views.py index 560cec98..b080a56e 100644 --- a/apps/fyle/views.py +++ b/apps/fyle/views.py @@ -15,7 +15,7 @@ from apps.workspaces.models import Configuration, FyleCredential, Workspace -from .tasks import schedule_expense_group_creation +from .tasks import schedule_expense_group_creation, get_task_log_and_fund_source, create_expense_groups from .helpers import check_interval_and_sync_dimension, sync_dimensions from .models import Expense, ExpenseGroup, ExpenseGroupSettings, ExpenseFilter from .serializers import ExpenseGroupSerializer, ExpenseSerializer, ExpenseFieldSerializer, \ @@ -408,3 +408,20 @@ def get(self, request, *args, **kwargs): }, status=status.HTTP_400_BAD_REQUEST ) + + +class ExpenseGroupSyncView(generics.CreateAPIView): + """ + Create expense groups + """ + def post(self, request, *args, **kwargs): + """ + Post expense groups creation + """ + task_log, fund_source, configuration = get_task_log_and_fund_source(kwargs['workspace_id']) + + create_expense_groups(kwargs['workspace_id'], configuration ,fund_source, task_log) + + return Response( + status=status.HTTP_200_OK + ) diff --git a/apps/netsuite/tasks.py b/apps/netsuite/tasks.py index e389c34c..09e55574 100644 --- a/apps/netsuite/tasks.py +++ b/apps/netsuite/tasks.py @@ -12,6 +12,7 @@ from apps.netsuite.exceptions import handle_netsuite_exceptions from django_q.models import Schedule from django_q.tasks import Chain, async_task +from fyle_netsuite_api.utils import generate_netsuite_export_url from netsuitesdk.internal.exceptions import NetSuiteRequestError from netsuitesdk import NetSuiteRateLimitError, NetSuiteLoginError @@ -444,10 +445,12 @@ def create_bill(expense_group, task_log_id, last_export): expense_group.exported_at = datetime.now() expense_group.response_logs = created_bill + expense_group.url = generate_netsuite_export_url(response_logs=created_bill, ns_account_id=netsuite_credentials) + expense_group.save() + resolve_errors_for_exported_expense_group(expense_group) - task_log.save() @handle_netsuite_exceptions(payment=False) @@ -513,6 +516,7 @@ def create_credit_card_charge(expense_group, task_log_id, last_export): expense_group.exported_at = datetime.now() expense_group.response_logs = created_credit_card_charge + expense_group.export_url = generate_netsuite_export_url(response_logs=created_credit_card_charge, ns_account_id=netsuite_credentials) expense_group.save() resolve_errors_for_exported_expense_group(expense_group) @@ -558,10 +562,10 @@ def create_expense_report(expense_group, task_log_id, last_export): expense_group.exported_at = datetime.now() expense_group.response_logs = created_expense_report + expense_group.export_url = generate_netsuite_export_url(response_logs=created_expense_report, ns_account_id=netsuite_credentials) expense_group.save() resolve_errors_for_exported_expense_group(expense_group) - task_log.save() @handle_netsuite_exceptions(payment=False) @@ -606,11 +610,10 @@ def create_journal_entry(expense_group, task_log_id, last_export): expense_group.exported_at = datetime.now() expense_group.response_logs = created_journal_entry + expense_group.export_url = generate_netsuite_export_url(response_logs=created_journal_entry, ns_account_id=netsuite_credentials) expense_group.save() resolve_errors_for_exported_expense_group(expense_group) - - task_log.save() - + def __validate_general_mapping(expense_group: ExpenseGroup, configuration: Configuration) -> List[BulkError]: bulk_errors = [] diff --git a/fyle_netsuite_api/utils.py b/fyle_netsuite_api/utils.py index e973f5ab..acf9cb77 100644 --- a/fyle_netsuite_api/utils.py +++ b/fyle_netsuite_api/utils.py @@ -1,6 +1,17 @@ from rest_framework.views import Response from rest_framework.serializers import ValidationError +import logging +logger = logging.getLogger(__name__) +logger.level = logging.INFO + +EXPORT_TYPE_REDIRECTION = { + 'vendorBill': 'vendbill', + 'expenseReport': 'exprept', + 'journalEntry': 'journal', + 'chargeCard': 'cardchrg', + 'chargeCardRefund': 'cardrfnd' +} def assert_valid(condition: bool, message: str) -> Response or None: """ @@ -23,3 +34,16 @@ def filter_queryset(self, queryset): filter_kwargs = {self.lookup_field: lookup_value} queryset = queryset.filter(**filter_kwargs) return super().filter_queryset(queryset) + + +def generate_netsuite_export_url(response_logs, ns_account_id): + if response_logs: + try: + export_type = response_logs['type'] if response_logs['type'] else 'chargeCard' + internal_id = response_logs['internalId'] + redirection = EXPORT_TYPE_REDIRECTION[export_type] + url = f'https://{ns_account_id}.app.netsuite.com/app/accounting/transactions/${redirection}.nl?id={internal_id}' + return url + except Exception as exception: + logger.exception({'error': exception}) + return None diff --git a/scripts/python/update-export-url.py b/scripts/python/update-export-url.py new file mode 100644 index 00000000..53b85be1 --- /dev/null +++ b/scripts/python/update-export-url.py @@ -0,0 +1,20 @@ +from apps.fyle.models import ExpenseGroup +from apps.workspaces.models import NetSuiteCredentials, Workspace +from fyle_netsuite_api.utils import generate_netsuite_export_url + + +prod_workspaces = Workspace.objects.exclude( + name__iregex=r'(fyle|test)', +) + +for workspace in prod_workspaces: + page_size = 200 + expense_group_counts = ExpenseGroup.objects.filter(workspace_id=workspace.id, response_logs__isnull=False).count() + for offset in range(0, expense_group_counts, page_size): + expense_to_be_updated = [] + limit = offset + page_size + paginated_expense_groups = ExpenseGroup.objects.filter(workspace_id=workspace.id, response_logs__isnull=False)[offset:limit] + for expense_group in paginated_expense_groups: + netsuite_cred = NetSuiteCredentials.objects.get(workspace_id=workspace.id) + expense_group.export_url = generate_netsuite_export_url(response_logs=expense_group.response_logs, ns_account_id=netsuite_cred.ns_account_id) + expense_group.save() diff --git a/scripts/sql/scripts/021-fill-employee_name-in-expenses-and-expense_groups.sql b/scripts/sql/scripts/021-fill-employee_name-in-expenses-and-expense_groups.sql new file mode 100644 index 00000000..b92355e5 --- /dev/null +++ b/scripts/sql/scripts/021-fill-employee_name-in-expenses-and-expense_groups.sql @@ -0,0 +1,29 @@ +rollback; +begin; + +with ws as ( + select expense_attributes.detail->>'full_name' as expense_attributes_full_name, + expense_attributes.workspace_id as expense_attributes_workspace_id, + expense_attributes.value as expense_attribute_email + from expense_groups + inner join expense_attributes on expense_attributes.value = expense_groups.description->>'employee_email' + where expense_groups.workspace_id = expense_attributes.workspace_id +) + +update expense_groups +set employee_name = ws.expense_attributes_full_name +from ws +where expense_groups.description->>'employee_email' = ws.expense_attribute_email; + + +-- Run this in after running the above query. +with ex as ( + select expense_groups.employee_name as employee_name + from expense_groups + inner join expense_groups_expenses on expense_groups.id = expense_groups_expenses.expensegroup_id + inner join expenses on expense_groups_expenses.expense_id = expenses.id +) + +update expenses +set employee_name = ex.employee_name +from ex; diff --git a/tests/sql_fixtures/reset_db_fixtures/reset_db.sql b/tests/sql_fixtures/reset_db_fixtures/reset_db.sql index dcac805b..f0ca792c 100644 --- a/tests/sql_fixtures/reset_db_fixtures/reset_db.sql +++ b/tests/sql_fixtures/reset_db_fixtures/reset_db.sql @@ -2,7 +2,7 @@ -- PostgreSQL database dump -- --- Dumped from database version 15.4 (Debian 15.4-2.pgdg120+1) +-- Dumped from database version 15.5 (Debian 15.5-1.pgdg120+1) -- Dumped by pg_dump version 15.5 (Debian 15.5-1.pgdg120+1) SET statement_timeout = 0; @@ -927,7 +927,9 @@ CREATE TABLE public.expense_groups ( workspace_id integer NOT NULL, fund_source character varying(255) NOT NULL, exported_at timestamp with time zone, - response_logs jsonb + response_logs jsonb, + employee_name character varying(100), + export_url character varying(255) ); @@ -7826,10 +7828,12 @@ COPY public.django_migrations (id, app, name, applied) FROM stdin; 169 mappings 0010_auto_20231025_0915 2023-11-07 07:21:37.268291+00 170 mappings 0011_auto_20231107_0720 2023-11-07 07:21:37.285191+00 171 netsuite 0023_bill_department_id 2023-11-07 07:21:37.291269+00 -172 workspaces 0036_auto_20231027_0709 2023-11-20 12:12:44.910371+00 -173 tasks 0009_error 2023-11-20 12:12:44.97278+00 -174 workspaces 0037_lastexportdetail 2023-11-20 12:12:45.043763+00 -175 workspaces 0038_configuration_allow_intercompany_vendors 2023-11-28 10:23:29.709496+00 +172 workspaces 0037_lastexportdetail 2023-11-20 12:12:45.043763+00 +173 workspaces 0038_configuration_allow_intercompany_vendors 2023-11-28 10:23:29.709496+00 +174 fyle 0027_expensegroup_employee_name 2023-11-29 11:09:45.601313+00 +175 workspaces 0036_auto_20231027_0709 2023-11-20 11:19:47.53547+00 +176 tasks 0009_error 2023-11-20 11:19:47.609035+00 +177 fyle 0028_expensegroup_export_url 2023-11-22 11:49:48.090718+00 \. @@ -11419,13 +11423,13 @@ COPY public.expense_group_settings (id, reimbursable_expense_group_fields, corpo -- Data for Name: expense_groups; Type: TABLE DATA; Schema: public; Owner: postgres -- -COPY public.expense_groups (id, description, created_at, updated_at, workspace_id, fund_source, exported_at, response_logs) FROM stdin; -1 {"report_id": "rpuN3bgphxbK", "fund_source": "PERSONAL", "claim_number": "C/2021/11/R/5", "employee_email": "ashwin.t@fyle.in"} 2021-11-15 10:29:07.618062+00 2021-11-15 11:02:55.125634+00 1 PERSONAL \N \N -2 {"report_id": "rpHLA9Dfp9hN", "fund_source": "CCC", "claim_number": "C/2021/11/R/6", "employee_email": "ashwin.t@fyle.in"} 2021-11-15 13:12:12.275539+00 2021-11-15 13:27:27.538211+00 1 CCC \N \N -3 {"report_id": "rpu5W0LYrk6e", "fund_source": "PERSONAL", "claim_number": "C/2021/11/R/2", "employee_email": "ashwin.t@fyle.in"} 2021-11-16 04:25:49.206777+00 2021-11-16 04:25:49.206809+00 2 PERSONAL \N \N -4 {"spent_at": "2021-11-16", "report_id": "rprqDvARHUnv", "expense_id": "txMLGb6Xy8m8", "fund_source": "CCC", "claim_number": "C/2021/11/R/1", "employee_email": "ashwin.t@fyle.in"} 2021-11-16 04:25:49.226855+00 2021-11-16 04:25:49.226855+00 2 CCC \N \N -47 {"report_id": "rpXqCutQj85N", "fund_source": "PERSONAL", "claim_number": "C/2021/12/R/1", "employee_email": "admin1@fyleforintacct.in"} 2021-12-03 11:26:58.731339+00 2021-12-03 11:26:58.731398+00 49 PERSONAL \N \N -48 {"report_id": "rpXqCutQj85N", "expense_id": "txcKVVELn1Vl", "fund_source": "CCC", "claim_number": "C/2021/12/R/1", "employee_email": "admin1@fyleforintacct.in"} 2021-12-03 11:26:58.746214+00 2021-12-03 11:26:58.746248+00 49 CCC \N \N +COPY public.expense_groups (id, description, created_at, updated_at, workspace_id, fund_source, exported_at, response_logs, employee_name, export_url) FROM stdin; +1 {"report_id": "rpuN3bgphxbK", "fund_source": "PERSONAL", "claim_number": "C/2021/11/R/5", "employee_email": "ashwin.t@fyle.in"} 2021-11-15 10:29:07.618062+00 2021-11-15 11:02:55.125634+00 1 PERSONAL \N \N \N \N +2 {"report_id": "rpHLA9Dfp9hN", "fund_source": "CCC", "claim_number": "C/2021/11/R/6", "employee_email": "ashwin.t@fyle.in"} 2021-11-15 13:12:12.275539+00 2021-11-15 13:27:27.538211+00 1 CCC \N \N \N \N +3 {"report_id": "rpu5W0LYrk6e", "fund_source": "PERSONAL", "claim_number": "C/2021/11/R/2", "employee_email": "ashwin.t@fyle.in"} 2021-11-16 04:25:49.206777+00 2021-11-16 04:25:49.206809+00 2 PERSONAL \N \N \N \N +4 {"spent_at": "2021-11-16", "report_id": "rprqDvARHUnv", "expense_id": "txMLGb6Xy8m8", "fund_source": "CCC", "claim_number": "C/2021/11/R/1", "employee_email": "ashwin.t@fyle.in"} 2021-11-16 04:25:49.226855+00 2021-11-16 04:25:49.226855+00 2 CCC \N \N \N \N +47 {"report_id": "rpXqCutQj85N", "fund_source": "PERSONAL", "claim_number": "C/2021/12/R/1", "employee_email": "admin1@fyleforintacct.in"} 2021-12-03 11:26:58.731339+00 2021-12-03 11:26:58.731398+00 49 PERSONAL \N \N \N \N +48 {"report_id": "rpXqCutQj85N", "expense_id": "txcKVVELn1Vl", "fund_source": "CCC", "claim_number": "C/2021/12/R/1", "employee_email": "admin1@fyleforintacct.in"} 2021-12-03 11:26:58.746214+00 2021-12-03 11:26:58.746248+00 49 CCC \N \N \N \N \. @@ -11725,7 +11729,7 @@ SELECT pg_catalog.setval('public.django_content_type_id_seq', 45, true); -- Name: django_migrations_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres -- -SELECT pg_catalog.setval('public.django_migrations_id_seq', 175, true); +SELECT pg_catalog.setval('public.django_migrations_id_seq', 176, true); -- diff --git a/tests/test_fyle/fixtures.py b/tests/test_fyle/fixtures.py index 24ba5d33..81690a29 100644 --- a/tests/test_fyle/fixtures.py +++ b/tests/test_fyle/fixtures.py @@ -131,7 +131,14 @@ 'claim_number': ' C/2021/12/R/198', 'employee_email': 'jhonsnow@fyle.in', 'employee_name': None, - 'fund_source': 'CCC' + 'fund_source': 'CCC', + 'expense_number': 'E/2023/09/T/7', + 'vendor': None, + 'category': 'Train', + 'amount': 101.0, + 'report_id': 'dummy_report_id', + 'settlement_id': 'dummy_settlement_id', + 'expense_id': 'dummy_expense_id' }, { 'updated_at': '2021-12-03T11:26:58.702209Z', @@ -529,6 +536,7 @@ "expense_group_id": { "id": 1, "fund_source": "PERSONAL", + 'employee_name': 'Ashwin', "description": { "report_id": "rpuN3bgphxbK", "fund_source": "PERSONAL", @@ -541,11 +549,25 @@ "externalId": "03294720937402397402937", "internalId": "116142", }, + "export_url": "https://TSTDRV2089588.app.netsuite.com/app/accounting/transactions/$journal.nl?id=725456", "created_at": "2021-11-15T10:29:07.618062Z", "exported_at": "2021-11-15T11:02:55.125205Z", "updated_at": "2021-11-15T11:02:55.125634Z", "workspace": 1, - "expenses": [1], + "expenses": [ { + "updated_at": "2023-11-21T10:41:09.919000Z", + "claim_number": "C/2023/06/R/4", + "employee_email": "admin1@fyleforbadassashu.in", + "employee_name": "Theresa Brown", + "fund_source": "PERSONAL", + "expense_number": "E/2023/06/T/21", + "vendor": "95110", + "category": "Airlines", + "amount": 6377.0, + "report_id": "rp0kaXoqkJle", + "settlement_id": "setPzkM7eyQFd", + "expense_id": "txyZ1zJDQfiK" + }], }, "expense_group_setting_response": { "id": 1, @@ -54964,4 +54986,4 @@ ] } ], -} \ No newline at end of file +}