added field export_url in expense group and util to generate URL (#457)

* add employee name in expense and script to populate data

* test fixture changes

* added field export_url in expense group and util to generate URL

* updated test and fixtures

* changed scripts to batch update export url

* bug fix

* comment resolved

* added more fields in expense serializer for redirection (#458)

* added more fields in expense serializer

* Sync import API (#459)

* added expense group sync API

* minor changes

* added url for expense group sync view

apps/fyle/migrations/0028_expensegroup_export_url.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.14 on 2023-11-22 10:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('fyle', '0027_expensegroup_employee_name'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='expensegroup',
+            name='export_url',
+            field=models.CharField(help_text='Netsuite URL for the exported expenses', max_length=255, null=True),
+        ),
+    ]

apps/fyle/models.py

@@ -328,6 +328,7 @@ class ExpenseGroup(models.Model):
     employee_name = models.CharField(max_length=100, help_text='Expense Group Employee Name', null=True)
     description = JSONField(max_length=255, help_text='Description', null=True)
     response_logs = JSONField(help_text='Reponse log of the export', null=True)
+    export_url = models.CharField(max_length=255, help_text='Netsuite URL for the exported expenses', null=True)
     created_at = models.DateTimeField(auto_now_add=True, help_text='Created at')
     exported_at = models.DateTimeField(help_text='Exported at', null=True)
     updated_at = models.DateTimeField(auto_now=True, help_text='Updated at')
@@ -393,6 +394,10 @@ def create_expense_groups_by_report_id_fund_source(expense_objects: List[Expense
             if expense_group_settings.ccc_export_date_type == 'last_spent_at':
                 expense_group['last_spent_at'] = Expense.objects.filter(
                     id__in=expense_group['expense_ids']).order_by('-spent_at').first().spent_at
+
+            employee_name = Expense.objects.filter(
+                id__in=expense_group['expense_ids']
+            ).first().employee_name
 
             employee_name = Expense.objects.filter(
                 id__in=expense_group['expense_ids']

apps/fyle/serializers.py

@@ -5,13 +5,24 @@
 from .models import Expense, ExpenseFilter, ExpenseGroup, ExpenseGroupSettings
 
 
+class ExpenseSerializer(serializers.ModelSerializer):
+    """
+    Expense serializer
+    """
+    class Meta:
+        model = Expense
+        fields = ['updated_at', 'claim_number', 'employee_email', 'employee_name', 'fund_source', 'expense_number', 'vendor', 'category', 'amount',
+        'report_id', 'settlement_id', 'expense_id']
+
 class ExpenseGroupSerializer(serializers.ModelSerializer):
     """
     Expense group serializer
     """
+    expenses = ExpenseSerializer(many=True)
     class Meta:
         model = ExpenseGroup
         fields = '__all__'
+        extra_fields = ['expenses']
 
 
 class ExpenseGroupExpenseSerializer(serializers.ModelSerializer):
@@ -60,12 +71,3 @@ def create(self, validated_data):
         )
 
         return expense_filter
-
-
-class ExpenseSerializer(serializers.ModelSerializer):
-    """
-    Expense serializer
-    """
-    class Meta:
-        model = Expense
-        fields = ['updated_at', 'claim_number', 'employee_email', 'employee_name', 'fund_source']

apps/fyle/tasks.py

@@ -24,27 +24,32 @@
     'CCC': 'PERSONAL_CORPORATE_CREDIT_CARD_ACCOUNT'
 }
 
-def schedule_expense_group_creation(workspace_id: int):
-    """
-    Schedule Expense group creation
-    :param workspace_id: Workspace id
-    :param user: User email
-    :return: None
-    """
+def get_task_log_and_fund_source(workspace_id: int):
     task_log, _ = TaskLog.objects.update_or_create(
         workspace_id=workspace_id,
         type='FETCHING_EXPENSES',
         defaults={
-            'status': 'IN_PROGRESS'
+           'status': 'IN_PROGRESS'
         }
     )
 
     configuration = Configuration.objects.get(workspace_id=workspace_id)
-
     fund_source = ['PERSONAL']
     if configuration.corporate_credit_card_expenses_object is not None:
         fund_source.append('CCC')
 
+    return task_log, fund_source, configuration
+
+def schedule_expense_group_creation(workspace_id: int):
+    """
+    Schedule Expense group creation
+    :param workspace_id: Workspace id
+    :param user: User email
+    :return: None
+    """
+
+    task_log, fund_source, configuration = get_task_log_and_fund_source(workspace_id)
+
     async_task('apps.fyle.tasks.create_expense_groups', workspace_id, configuration, fund_source, task_log)
 
 

apps/fyle/urls.py

@@ -2,7 +2,7 @@
 
 from django.urls import path
 
-from .views import ExpenseGroupView, ExpenseGroupByIdView, ExpenseGroupScheduleView, ExportableExpenseGroupsView, FyleFieldsView, ExpenseView,\
+from .views import ExpenseGroupSyncView, ExpenseGroupView, ExpenseGroupByIdView, ExpenseGroupScheduleView, ExportableExpenseGroupsView, FyleFieldsView, ExpenseView,\
     ExpenseAttributesView, ExpenseGroupSettingsView, SyncFyleDimensionView, RefreshFyleDimensionView,\
     ExpenseGroupCountView, ExpenseFilterView, ExpenseGroupExpenseView, CustomFieldView
 
@@ -13,7 +13,8 @@
     path('expense_groups/<int:pk>/', ExpenseGroupByIdView.as_view(), name='expense-group-by-id'),
     path('expense_groups/<int:expense_group_id>/expenses/', ExpenseGroupExpenseView.as_view(), name='expense-group-expenses'),
     path('expense_group_settings/', ExpenseGroupSettingsView.as_view(), name='expense-group-settings'),
-    path('exportable_expense_groups/', ExportableExpenseGroupsView.as_view(), name='expense-expense-groups')
+    path('exportable_expense_groups/', ExportableExpenseGroupsView.as_view(), name='expense-expense-groups'),
+    path('expense_groups/sync/', ExpenseGroupSyncView.as_view(), name='sync-expense-groups'),
 ]
 
 fyle_dimension_paths = [

apps/fyle/views.py

@@ -15,7 +15,7 @@
 
 from apps.workspaces.models import Configuration, FyleCredential, Workspace
 
-from .tasks import schedule_expense_group_creation
+from .tasks import schedule_expense_group_creation, get_task_log_and_fund_source, create_expense_groups
 from .helpers import check_interval_and_sync_dimension, sync_dimensions
 from .models import Expense, ExpenseGroup, ExpenseGroupSettings, ExpenseFilter
 from .serializers import ExpenseGroupSerializer, ExpenseSerializer, ExpenseFieldSerializer, \
@@ -408,3 +408,20 @@ def get(self, request, *args, **kwargs):
                 },
                 status=status.HTTP_400_BAD_REQUEST
             )
+
+
+class ExpenseGroupSyncView(generics.CreateAPIView):
+    """
+    Create expense groups
+    """
+    def post(self, request, *args, **kwargs):
+        """
+        Post expense groups creation
+        """
+        task_log, fund_source, configuration = get_task_log_and_fund_source(kwargs['workspace_id'])
+
+        create_expense_groups(kwargs['workspace_id'], configuration ,fund_source, task_log)
+
+        return Response(
+            status=status.HTTP_200_OK
+        )

apps/netsuite/tasks.py

@@ -12,6 +12,7 @@
 from apps.netsuite.exceptions import handle_netsuite_exceptions
 from django_q.models import Schedule
 from django_q.tasks import Chain, async_task
+from fyle_netsuite_api.utils import generate_netsuite_export_url
 
 from netsuitesdk.internal.exceptions import NetSuiteRequestError
 from netsuitesdk import NetSuiteRateLimitError, NetSuiteLoginError
@@ -444,10 +445,12 @@ def create_bill(expense_group, task_log_id, last_export):
 
         expense_group.exported_at = datetime.now()
         expense_group.response_logs = created_bill
+        expense_group.url = generate_netsuite_export_url(response_logs=created_bill, ns_account_id=netsuite_credentials)
+
         expense_group.save()
+
         resolve_errors_for_exported_expense_group(expense_group)
 
-        task_log.save()
 
 
 @handle_netsuite_exceptions(payment=False)
@@ -513,6 +516,7 @@ def create_credit_card_charge(expense_group, task_log_id, last_export):
 
         expense_group.exported_at = datetime.now()
         expense_group.response_logs = created_credit_card_charge
+        expense_group.export_url = generate_netsuite_export_url(response_logs=created_credit_card_charge, ns_account_id=netsuite_credentials)
         expense_group.save()
         resolve_errors_for_exported_expense_group(expense_group)
 
@@ -558,10 +562,10 @@ def create_expense_report(expense_group, task_log_id, last_export):
 
         expense_group.exported_at = datetime.now()
         expense_group.response_logs = created_expense_report
+        expense_group.export_url = generate_netsuite_export_url(response_logs=created_expense_report, ns_account_id=netsuite_credentials)
         expense_group.save()
         resolve_errors_for_exported_expense_group(expense_group)
 
-        task_log.save()
 
 
 @handle_netsuite_exceptions(payment=False)
@@ -606,11 +610,10 @@ def create_journal_entry(expense_group, task_log_id, last_export):
 
         expense_group.exported_at = datetime.now()
         expense_group.response_logs = created_journal_entry
+        expense_group.export_url = generate_netsuite_export_url(response_logs=created_journal_entry, ns_account_id=netsuite_credentials)      
         expense_group.save()
         resolve_errors_for_exported_expense_group(expense_group)
-
-        task_log.save()
-
+
 
 def __validate_general_mapping(expense_group: ExpenseGroup, configuration: Configuration) -> List[BulkError]:
     bulk_errors = []

fyle_netsuite_api/utils.py

@@ -1,6 +1,17 @@
 from rest_framework.views import Response
 from rest_framework.serializers import ValidationError
+import logging
 
+logger = logging.getLogger(__name__)
+logger.level = logging.INFO
+
+EXPORT_TYPE_REDIRECTION = {
+    'vendorBill': 'vendbill',
+    'expenseReport': 'exprept',
+    'journalEntry': 'journal',
+    'chargeCard': 'cardchrg',
+    'chargeCardRefund': 'cardrfnd'
+}
 
 def assert_valid(condition: bool, message: str) -> Response or None:
     """
@@ -23,3 +34,16 @@ def filter_queryset(self, queryset):
             filter_kwargs = {self.lookup_field: lookup_value}
             queryset = queryset.filter(**filter_kwargs)
         return super().filter_queryset(queryset)
+
+
+def generate_netsuite_export_url(response_logs, ns_account_id):
+    if response_logs:
+        try:
+            export_type = response_logs['type'] if response_logs['type'] else 'chargeCard'
+            internal_id = response_logs['internalId']
+            redirection = EXPORT_TYPE_REDIRECTION[export_type]
+            url = f'https://{ns_account_id}.app.netsuite.com/app/accounting/transactions/${redirection}.nl?id={internal_id}'
+            return url
+        except Exception as exception:
+            logger.exception({'error': exception})
+    return None

scripts/python/update-export-url.py

@@ -0,0 +1,20 @@
+from apps.fyle.models import ExpenseGroup
+from apps.workspaces.models import NetSuiteCredentials, Workspace
+from fyle_netsuite_api.utils import generate_netsuite_export_url
+
+
+prod_workspaces = Workspace.objects.exclude(
+    name__iregex=r'(fyle|test)',
+)
+
+for workspace in prod_workspaces:
+    page_size = 200
+    expense_group_counts = ExpenseGroup.objects.filter(workspace_id=workspace.id, response_logs__isnull=False).count()
+    for offset in range(0, expense_group_counts, page_size):
+        expense_to_be_updated = []
+        limit = offset + page_size
+        paginated_expense_groups = ExpenseGroup.objects.filter(workspace_id=workspace.id, response_logs__isnull=False)[offset:limit]
+        for expense_group in paginated_expense_groups:
+            netsuite_cred = NetSuiteCredentials.objects.get(workspace_id=workspace.id)
+            expense_group.export_url = generate_netsuite_export_url(response_logs=expense_group.response_logs, ns_account_id=netsuite_cred.ns_account_id)
+            expense_group.save()

scripts/sql/scripts/020-fill-employee_name-in-expenses-and-expense_groups.sql

@@ -0,0 +1,29 @@
+rollback;
+begin;
+
+with ws as (
+  select expense_attributes.detail->>'full_name' as expense_attributes_full_name,
+    expense_attributes.workspace_id as expense_attributes_workspace_id,
+    expense_attributes.value as expense_attribute_email
+  from expense_groups
+    inner join expense_attributes on expense_attributes.value = expense_groups.description->>'employee_email'
+  where expense_groups.workspace_id = expense_attributes.workspace_id
+)
+
+update expense_groups
+set employee_name = ws.expense_attributes_full_name
+from ws
+where expense_groups.description->>'employee_email' = ws.expense_attribute_email;
+
+
+-- Run this in after running the above query.
+with ex as (
+  select expense_groups.employee_name as employee_name
+  from expense_groups
+    inner join expense_groups_expenses on expense_groups.id = expense_groups_expenses.expensegroup_id
+    inner join expenses on expense_groups_expenses.expense_id = expenses.id
+)
+
+update expenses
+set employee_name = ex.employee_name
+from ex;

tests/sql_fixtures/reset_db_fixtures/reset_db.sql

@@ -928,7 +928,8 @@ CREATE TABLE public.expense_groups (
     fund_source character varying(255) NOT NULL,
     exported_at timestamp with time zone,
     response_logs jsonb,
-    employee_name character varying(100)
+    employee_name character varying(100),
+    export_url character varying(255)
 );
 
 
@@ -7827,11 +7828,12 @@ COPY public.django_migrations (id, app, name, applied) FROM stdin;
 169	mappings	0010_auto_20231025_0915	2023-11-07 07:21:37.268291+00
 170	mappings	0011_auto_20231107_0720	2023-11-07 07:21:37.285191+00
 171	netsuite	0023_bill_department_id	2023-11-07 07:21:37.291269+00
-172	workspaces	0036_auto_20231027_0709	2023-11-20 12:12:44.910371+00
-173	tasks	0009_error	2023-11-20 12:12:44.97278+00
-174	workspaces	0037_lastexportdetail	2023-11-20 12:12:45.043763+00
-175	workspaces	0038_configuration_allow_intercompany_vendors	2023-11-28 10:23:29.709496+00
-176	fyle	0027_expensegroup_employee_name	2023-11-29 11:09:45.601313+00
+172	workspaces	0037_lastexportdetail	2023-11-20 12:12:45.043763+00
+173	workspaces	0038_configuration_allow_intercompany_vendors	2023-11-28 10:23:29.709496+00
+174	fyle	0027_expensegroup_employee_name	2023-11-29 11:09:45.601313+00
+175	workspaces	0036_auto_20231027_0709	2023-11-20 11:19:47.53547+00
+176	tasks	0009_error	2023-11-20 11:19:47.609035+00
+177	fyle	0028_expensegroup_export_url	2023-11-22 11:49:48.090718+00
 \.
 
 
@@ -11421,13 +11423,13 @@ COPY public.expense_group_settings (id, reimbursable_expense_group_fields, corpo
 -- Data for Name: expense_groups; Type: TABLE DATA; Schema: public; Owner: postgres
 --
 
-COPY public.expense_groups (id, description, created_at, updated_at, workspace_id, fund_source, exported_at, response_logs, employee_name) FROM stdin;
-1	{"report_id": "rpuN3bgphxbK", "fund_source": "PERSONAL", "claim_number": "C/2021/11/R/5", "employee_email": "[email protected]"}	2021-11-15 10:29:07.618062+00	2021-11-15 11:02:55.125634+00	1	PERSONAL	\N	\N	\N
-2	{"report_id": "rpHLA9Dfp9hN", "fund_source": "CCC", "claim_number": "C/2021/11/R/6", "employee_email": "[email protected]"}	2021-11-15 13:12:12.275539+00	2021-11-15 13:27:27.538211+00	1	CCC	\N	\N	\N
-3	{"report_id": "rpu5W0LYrk6e", "fund_source": "PERSONAL", "claim_number": "C/2021/11/R/2", "employee_email": "[email protected]"}	2021-11-16 04:25:49.206777+00	2021-11-16 04:25:49.206809+00	2	PERSONAL	\N	\N	\N
-4	{"spent_at": "2021-11-16", "report_id": "rprqDvARHUnv", "expense_id": "txMLGb6Xy8m8", "fund_source": "CCC", "claim_number": "C/2021/11/R/1", "employee_email": "[email protected]"}	2021-11-16 04:25:49.226855+00	2021-11-16 04:25:49.226855+00	2	CCC	\N	\N	\N
-47	{"report_id": "rpXqCutQj85N", "fund_source": "PERSONAL", "claim_number": "C/2021/12/R/1", "employee_email": "[email protected]"}	2021-12-03 11:26:58.731339+00	2021-12-03 11:26:58.731398+00	49	PERSONAL	\N	\N	\N
-48	{"report_id": "rpXqCutQj85N", "expense_id": "txcKVVELn1Vl", "fund_source": "CCC", "claim_number": "C/2021/12/R/1", "employee_email": "[email protected]"}	2021-12-03 11:26:58.746214+00	2021-12-03 11:26:58.746248+00	49	CCC	\N	\N	\N
+COPY public.expense_groups (id, description, created_at, updated_at, workspace_id, fund_source, exported_at, response_logs, employee_name, export_url) FROM stdin;
+1	{"report_id": "rpuN3bgphxbK", "fund_source": "PERSONAL", "claim_number": "C/2021/11/R/5", "employee_email": "[email protected]"}	2021-11-15 10:29:07.618062+00	2021-11-15 11:02:55.125634+00	1	PERSONAL	\N	\N	\N	\N
+2	{"report_id": "rpHLA9Dfp9hN", "fund_source": "CCC", "claim_number": "C/2021/11/R/6", "employee_email": "[email protected]"}	2021-11-15 13:12:12.275539+00	2021-11-15 13:27:27.538211+00	1	CCC	\N	\N	\N	\N
+3	{"report_id": "rpu5W0LYrk6e", "fund_source": "PERSONAL", "claim_number": "C/2021/11/R/2", "employee_email": "[email protected]"}	2021-11-16 04:25:49.206777+00	2021-11-16 04:25:49.206809+00	2	PERSONAL	\N	\N	\N	\N
+4	{"spent_at": "2021-11-16", "report_id": "rprqDvARHUnv", "expense_id": "txMLGb6Xy8m8", "fund_source": "CCC", "claim_number": "C/2021/11/R/1", "employee_email": "[email protected]"}	2021-11-16 04:25:49.226855+00	2021-11-16 04:25:49.226855+00	2	CCC	\N	\N	\N	\N
+47	{"report_id": "rpXqCutQj85N", "fund_source": "PERSONAL", "claim_number": "C/2021/12/R/1", "employee_email": "[email protected]"}	2021-12-03 11:26:58.731339+00	2021-12-03 11:26:58.731398+00	49	PERSONAL	\N	\N	\N	\N
+48	{"report_id": "rpXqCutQj85N", "expense_id": "txcKVVELn1Vl", "fund_source": "CCC", "claim_number": "C/2021/12/R/1", "employee_email": "[email protected]"}	2021-12-03 11:26:58.746214+00	2021-12-03 11:26:58.746248+00	49	CCC	\N	\N	\N	\N
 \.