Last Export Detail Table and API (#456)

apps/netsuite/tasks.py

@@ -24,7 +24,7 @@
 from apps.fyle.models import ExpenseGroup, Expense, Reimbursement
 from apps.mappings.models import GeneralMapping, SubsidiaryMapping
 from apps.tasks.models import TaskLog
-from apps.workspaces.models import NetSuiteCredentials, FyleCredential, Configuration, Workspace
+from apps.workspaces.models import LastExportDetail, NetSuiteCredentials, FyleCredential, Configuration, Workspace
 
 from .models import Bill, BillLineitem, ExpenseReport, ExpenseReportLineItem, JournalEntry, JournalEntryLineItem, \
     VendorPayment, VendorPaymentLineitem, CreditCardCharge, CreditCardChargeLineItem
@@ -71,6 +71,27 @@
 }
 
 
+def update_last_export_details(workspace_id):
+    last_export_detail = LastExportDetail.objects.get(workspace_id=workspace_id)
+
+    failed_exports = TaskLog.objects.filter(
+        ~Q(type__in=['CREATING_VENDOR_PAYMENT','FETCHING_EXPENSES']), workspace_id=workspace_id, status__in=['FAILED', 'FATAL']
+    ).count()
+
+    successful_exports = TaskLog.objects.filter(
+        ~Q(type__in=['CREATING_VENDOR_PAYMENT', 'FETCHING_EXPENSES']),
+        workspace_id=workspace_id,
+        status='COMPLETE',
+        updated_at__gt=last_export_detail.last_exported_at
+    ).count()
+
+    last_export_detail.failed_expense_groups_count = failed_exports
+    last_export_detail.successful_expense_groups_count = successful_exports
+    last_export_detail.total_expense_groups_count = failed_exports + successful_exports
+    last_export_detail.save()
+
+    return last_export_detail
+
 def load_attachments(netsuite_connection: NetSuiteConnector, expense: Expense, expense_group: ExpenseGroup):
     """
     Get attachments from Fyle
@@ -401,7 +422,7 @@ def upload_attachments_and_update_export(expenses: List[Expense], task_log: Task
         )
 
 
-def create_bill(expense_group, task_log_id):
+def create_bill(expense_group, task_log_id, last_export):
     task_log = TaskLog.objects.get(id=task_log_id)
 
     if task_log.status not in ['IN_PROGRESS', 'COMPLETE']:
@@ -499,9 +520,12 @@ def create_bill(expense_group, task_log_id):
         task_log.status = 'FATAL'
         task_log.save()
         __log_error(task_log)
+
+    if last_export:
+        update_last_export_details(expense_group.workspace_id)
 
 
-def create_credit_card_charge(expense_group, task_log_id):
+def create_credit_card_charge(expense_group, task_log_id, last_export):
     task_log = TaskLog.objects.get(id=task_log_id)
 
     if task_log.status not in ['IN_PROGRESS', 'COMPLETE']:
@@ -611,9 +635,12 @@ def create_credit_card_charge(expense_group, task_log_id):
         task_log.status = 'FATAL'
         task_log.save()
         __log_error(task_log)
+
+    if last_export:
+        update_last_export_details(expense_group.workspace_id)
 
 
-def create_expense_report(expense_group, task_log_id):
+def create_expense_report(expense_group, task_log_id, last_export):
     task_log = TaskLog.objects.get(id=task_log_id)
 
     if task_log.status not in ['IN_PROGRESS', 'COMPLETE']:
@@ -706,9 +733,11 @@ def create_expense_report(expense_group, task_log_id):
         task_log.status = 'FATAL'
         task_log.save()
         __log_error(task_log)
+
+    if last_export:
+        update_last_export_details(expense_group.workspace_id)
 
-
-def create_journal_entry(expense_group, task_log_id):
+def create_journal_entry(expense_group, task_log_id, last_export):
     task_log = TaskLog.objects.get(id=task_log_id)
 
     if task_log.status not in ['IN_PROGRESS', 'COMPLETE']:
@@ -801,6 +830,9 @@ def create_journal_entry(expense_group, task_log_id):
         task_log.status = 'FATAL'
         task_log.save()
         __log_error(task_log)
+
+    if last_export:
+        update_last_export_details(expense_group.workspace_id)
 
 
 def __validate_general_mapping(expense_group: ExpenseGroup, configuration: Configuration) -> List[BulkError]:
@@ -1103,7 +1135,7 @@ def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str]):
         fyle_credentials = FyleCredential.objects.get(workspace_id=workspace_id)
         chain.append('apps.fyle.helpers.sync_dimensions', fyle_credentials, workspace_id)
 
-        for expense_group in expense_groups:
+        for index, expense_group in enumerate(expense_groups):
             task_log, _ = TaskLog.objects.get_or_create(
                 workspace_id=expense_group.workspace_id,
                 expense_group=expense_group,
@@ -1117,8 +1149,12 @@ def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str]):
                 task_log.type = 'CREATING_BILL'
                 task_log.status = 'ENQUEUED'
                 task_log.save()
+
+            last_export = False
+            if expense_groups.count() == index + 1:
+                last_export = True
 
-            chain.append('apps.netsuite.tasks.create_bill', expense_group, task_log.id)
+            chain.append('apps.netsuite.tasks.create_bill', expense_group, task_log.id, last_export)
 
             task_log.save()
         if chain.length() > 1:
@@ -1144,7 +1180,7 @@ def schedule_credit_card_charge_creation(workspace_id: int, expense_group_ids: L
         fyle_credentials = FyleCredential.objects.get(workspace_id=workspace_id)
         chain.append('apps.fyle.helpers.sync_dimensions', fyle_credentials, workspace_id)
 
-        for expense_group in expense_groups:
+        for index, expense_group in enumerate(expense_groups):
             expense_amount = expense_group.expenses.first().amount
             export_type = 'CREATING_CREDIT_CARD_CHARGE'
             if expense_amount < 0:
@@ -1163,8 +1199,12 @@ def schedule_credit_card_charge_creation(workspace_id: int, expense_group_ids: L
                 task_log.type = export_type
                 task_log.status = 'ENQUEUED'
                 task_log.save()
+
+            last_export = False
+            if expense_groups.count() == index + 1:
+                last_export = True
 
-            chain.append('apps.netsuite.tasks.create_credit_card_charge', expense_group, task_log.id)
+            chain.append('apps.netsuite.tasks.create_credit_card_charge', expense_group, task_log.id, last_export)
 
             task_log.save()
         if chain.length() > 1:
@@ -1190,7 +1230,7 @@ def schedule_expense_reports_creation(workspace_id: int, expense_group_ids: List
         fyle_credentials = FyleCredential.objects.get(workspace_id=workspace_id)
         chain.append('apps.fyle.helpers.sync_dimensions', fyle_credentials, workspace_id)
 
-        for expense_group in expense_groups:
+        for index, expense_group in enumerate(expense_groups):
             task_log, _ = TaskLog.objects.get_or_create(
                 workspace_id=expense_group.workspace_id,
                 expense_group=expense_group,
@@ -1204,8 +1244,12 @@ def schedule_expense_reports_creation(workspace_id: int, expense_group_ids: List
                 task_log.type = 'CREATING_EXPENSE_REPORT'
                 task_log.status = 'ENQUEUED'
                 task_log.save()
+
+            last_export = False
+            if expense_groups.count() == index + 1:
+                last_export = True
 
-            chain.append('apps.netsuite.tasks.create_expense_report', expense_group, task_log.id)
+            chain.append('apps.netsuite.tasks.create_expense_report', expense_group, task_log.id, last_export)
             task_log.save()
         if chain.length() > 1:
             chain.run()
@@ -1229,7 +1273,7 @@ def schedule_journal_entry_creation(workspace_id: int, expense_group_ids: List[s
         fyle_credentials = FyleCredential.objects.get(workspace_id=workspace_id)
         chain.append('apps.fyle.helpers.sync_dimensions', fyle_credentials, workspace_id)
 
-        for expense_group in expense_groups:
+        for index, expense_group in enumerate(expense_groups):
             task_log, _ = TaskLog.objects.get_or_create(
                 workspace_id=expense_group.workspace_id,
                 expense_group=expense_group,
@@ -1243,8 +1287,12 @@ def schedule_journal_entry_creation(workspace_id: int, expense_group_ids: List[s
                 task_log.type = 'CREATING_JOURNAL_ENTRY'
                 task_log.status = 'ENQUEUED'
                 task_log.save()
+
+            last_export = False
+            if expense_groups.count() == index + 1:
+                last_export = True
 
-            chain.append('apps.netsuite.tasks.create_journal_entry', expense_group, task_log.id)
+            chain.append('apps.netsuite.tasks.create_journal_entry', expense_group, task_log.id, last_export)
             task_log.save()
         if chain.length() > 1:
             chain.run()

apps/netsuite/urls.py

@@ -1,5 +1,7 @@
 from django.urls import path
 
+from apps.workspaces.views import LastExportDetailView
+
 from .views import NetSuiteFieldsView, DestinationAttributesView, CustomSegmentView, \
     SyncNetSuiteDimensionView, RefreshNetSuiteDimensionView, TriggerExportsView, TriggerPaymentsView, \
     DestinationAttributesCountView

apps/workspaces/migrations/0037_lastexportdetail.py

@@ -0,0 +1,32 @@
+# Generated by Django 3.1.14 on 2023-11-20 12:07
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('workspaces', '0036_auto_20231027_0709'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='LastExportDetail',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('last_exported_at', models.DateTimeField(help_text='Last exported at datetime', null=True)),
+                ('next_export', models.DateTimeField(help_text='next export datetime', null=True)),
+                ('export_mode', models.CharField(choices=[('MANUAL', 'MANUAL'), ('AUTO', 'AUTO')], help_text='Mode of the export Auto / Manual', max_length=50, null=True)),
+                ('total_expense_groups_count', models.IntegerField(help_text='Total count of expense groups exported', null=True)),
+                ('successful_expense_groups_count', models.IntegerField(help_text='count of successful expense_groups ', null=True)),
+                ('failed_expense_groups_count', models.IntegerField(help_text='count of failed expense_groups ', null=True)),
+                ('created_at', models.DateTimeField(auto_now_add=True, help_text='Created at datetime')),
+                ('updated_at', models.DateTimeField(auto_now=True, help_text='Updated at datetime')),
+                ('workspace', models.OneToOneField(help_text='Reference to Workspace model', on_delete=django.db.models.deletion.PROTECT, to='workspaces.workspace')),
+            ],
+            options={
+                'db_table': 'last_export_details',
+            },
+        ),
+    ]

apps/workspaces/models.py

@@ -182,3 +182,30 @@ class Configuration(models.Model):
 
     class Meta:
         db_table = 'configurations'
+
+
+EXPORT_MODE_CHOICES = (
+    ('MANUAL', 'MANUAL'),
+    ('AUTO', 'AUTO')
+)
+
+
+class LastExportDetail(models.Model):
+    """
+    Table to store Last Export Details
+    """
+    id = models.AutoField(primary_key=True)
+    last_exported_at = models.DateTimeField(help_text='Last exported at datetime', null=True)
+    next_export = models.DateTimeField(help_text='next export datetime', null=True)
+    export_mode = models.CharField(
+        max_length=50, help_text='Mode of the export Auto / Manual', choices=EXPORT_MODE_CHOICES, null=True
+    )
+    total_expense_groups_count = models.IntegerField(help_text='Total count of expense groups exported', null=True)
+    successful_expense_groups_count = models.IntegerField(help_text='count of successful expense_groups ', null=True)
+    failed_expense_groups_count = models.IntegerField(help_text='count of failed expense_groups ', null=True)
+    workspace = models.OneToOneField(Workspace, on_delete=models.PROTECT, help_text='Reference to Workspace model')
+    created_at = models.DateTimeField(auto_now_add=True, help_text='Created at datetime')
+    updated_at = models.DateTimeField(auto_now=True, help_text='Updated at datetime')
+
+    class Meta:
+        db_table = 'last_export_details'

apps/workspaces/serializers.py

@@ -3,7 +3,7 @@
 """
 from rest_framework import serializers
 
-from .models import Workspace, FyleCredential, NetSuiteCredentials, WorkspaceSchedule, Configuration
+from .models import LastExportDetail, Workspace, FyleCredential, NetSuiteCredentials, WorkspaceSchedule, Configuration
 
 
 class WorkspaceSerializer(serializers.ModelSerializer):
@@ -132,3 +132,12 @@ def validate(self, attrs):
     class Meta:
         model = Configuration
         fields = '__all__'
+
+
+class LastExportDetailSerializer(serializers.ModelSerializer):
+    """
+    Last export detail serializer
+    """
+    class Meta:
+        model = LastExportDetail
+        fields = '__all__'