Add direct export scripts

scripts/python/fill-accounting-export-summary.py

@@ -0,0 +1,99 @@
+from datetime import datetime
+
+from django.db.models import Q
+from django.conf import settings
+
+from apps.tasks.models import TaskLog
+from apps.workspaces.models import Workspace
+from apps.fyle.helpers import get_updated_accounting_export_summary
+from apps.fyle.models import Expense
+from apps.fyle.actions import __bulk_update_expenses
+
+# PLEASE RUN sql/scripts/022-fill-skipped-accounting-export-summary.sql BEFORE RUNNING THIS SCRIPT
+
+
+export_types = ['CREATING_JOURNAL_ENTRIES', 'CREATING_EXPENSE_REPORTS', 'CREATING_BILLS', 'CREATING_CHARGE_CARD_TRANSACTIONS']
+task_statuses = ['COMPLETE', 'FAILED', 'FATAL']
+
+
+# We'll handle all COMPLETE, ERROR expenses in this script
+workspaces = Workspace.objects.filter(
+    ~Q(name__icontains='fyle for') & ~Q(name__icontains='test')
+)
+
+start_time = datetime.now()
+number_of_expenses_without_accounting_export_summary = Expense.objects.filter(
+    accounting_export_summary__state__isnull=True
+).count()
+print('Number of expenses without accounting export summary - {}'.format(number_of_expenses_without_accounting_export_summary))
+for workspace in workspaces:
+    task_logs_count = TaskLog.objects.filter(
+        type__in=export_types,
+        workspace_id=workspace.id,
+        status__in=task_statuses
+    ).count()
+    print('Updating summary from workspace - {} with ID - {}'.format(workspace.name, workspace.id))
+    print('Number of task logs to be updated - {}'.format(task_logs_count))
+    page_size = 200
+    for offset in range(0, task_logs_count, page_size):
+        expense_to_be_updated = []
+        limit = offset + page_size
+        paginated_task_logs = TaskLog.objects.filter(
+            type__in=export_types,
+            workspace_id=workspace.id,
+            status__in=task_statuses
+        )[offset:limit]
+        for task_log in paginated_task_logs:
+            expense_group = task_log.expense_group
+            state = 'ERROR' if task_log.status == 'FAILED' or task_log.status == 'FATAL' else 'COMPLETE'
+            error_type = None
+            url = None
+            if task_log.status == 'FAILED' or task_log.status == 'FATAL':
+                for item in task_log.detail:
+                    if item.get('type') and item.get('type') == 'Category Mapping':
+                        error_type = 'MAPPING'
+                    else:
+                        error_type = 'ACCOUNTING_INTEGRATION_ERROR'
+                url = '{}/workspaces/main/dashboard'.format(settings.XERO_INTEGRATION_APP_URL)
+            else:
+                try:
+                    if task_log.type == 'CREATING_BILL':
+                        export_id = expense_group.response_logs['Invoices'][0]['InvoiceID']
+                        if workspace.xero_short_code:
+                            url = f'https://go.xero.com/organisationlogin/default.aspx?shortcode={workspace.xero_short_code}&redirecturl=/AccountsPayable/Edit.aspx?InvoiceID={export_id}'
+                        else:
+                            url = f'https://go.xero.com/AccountsPayable/View.aspx?invoiceID={export_id}'
+                    else:
+                        export_id = expense_group.response_logs['BankTransactions'][0]['BankTransactionID']
+                        account_id = expense_group.response_logs['BankTransactions'][0]['BankAccount']['AccountID']
+                        if workspace.xero_short_code:
+                            url = f'https://go.xero.com/organisationlogin/default.aspx?shortcode={workspace.xero_short_code}&redirecturl=/Bank/ViewTransaction.aspx?bankTransactionID={export_id}'
+                        else:
+                            url = f'https://go.xero.com/Bank/ViewTransaction.aspx?bankTransactionID={export_id}&accountID={account_id}'
+                except Exception as error:
+                    # Defaulting it to Intacct app url, worst case scenario if we're not able to parse it properly
+                    url = 'https://go.xero.com'
+            for expense in expense_group.expenses.filter(accounting_export_summary__state__isnull=True):
+                if url:
+                    expense_to_be_updated.append(
+                        Expense(
+                            id=expense.id,
+                            accounting_export_summary=get_updated_accounting_export_summary(
+                                expense.expense_id,
+                                state,
+                                error_type,
+                                url,
+                                False
+                            )
+                        )
+                    )
+        print('Updating {} expenses in batches of 50'.format(len(expense_to_be_updated)))
+        __bulk_update_expenses(expense_to_be_updated)
+
+
+number_of_expenses_without_accounting_export_summary = Expense.objects.filter(
+    accounting_export_summary__state__isnull=True
+).count()
+print('Number of expenses without accounting export summary - {}'.format(number_of_expenses_without_accounting_export_summary))
+end_time = datetime.now()
+print('Time taken - {}'.format(end_time - start_time))

scripts/python/post-accounting-export-summary.py

@@ -0,0 +1,43 @@
+from datetime import datetime
+from time import sleep
+
+from django.db.models import Q
+
+from apps.workspaces.models import Workspace
+from apps.fyle.models import Expense
+from apps.fyle.tasks import post_accounting_export_summary
+
+# PLEASE RUN scripts/python/fill-accounting-export-summary.py BEFORE RUNNING THIS SCRIPT
+workspaces = Workspace.objects.filter(
+    ~Q(name__icontains='fyle for') & ~Q(name__icontains='test')
+)
+
+start_time = datetime.now()
+number_of_expenses_to_be_posted = Expense.objects.filter(
+    accounting_export_summary__synced=False
+).count()
+print('Number of expenses to be posted - {}'.format(number_of_expenses_to_be_posted))
+for workspace in workspaces:
+    expenses_count = Expense.objects.filter(
+        accounting_export_summary__synced=False,
+        workspace_id=workspace.id
+    ).count()
+    print('Updating summary from workspace - {} with ID - {}'.format(workspace.name, workspace.id))
+    print('Number of expenses_count to be posted for the current workspace - {}'.format(expenses_count))
+    if expenses_count:
+        try:
+            sleep(1)
+            post_accounting_export_summary(workspace.fyle_org_id, workspace.id)
+        except Exception as e:
+            print('Error while posting accounting export summary for workspace - {} with ID - {}'.format(workspace.name, workspace.id))
+            print(e.__dict__)
+
+number_of_expenses_posted = Expense.objects.filter(
+    accounting_export_summary__synced=True
+).count()
+print('Number of expenses posted to Fyle - {}'.format(number_of_expenses_posted))
+end_time = datetime.now()
+print('Time taken - {}'.format(end_time - start_time))
+
+# This query should return 0 rows
+# select expense_group_id from task_logs where status not in ('ENQUEUED', 'IN_PROGRESS') and expense_group_id in (select expensegroup_id from expense_groups_expenses  where expense_id in (select id from expenses where accounting_export_summary ='{}'));

sql/scripts/022-fill-org-id-expenses.sql

@@ -0,0 +1,14 @@
+rollback;
+begin;
+
+with expense_groups as (
+    select w.fyle_org_id, e.id from expenses e
+    join expense_groups_expenses ege on e.id = ege.expense_id
+    join expense_groups eg on eg.id = ege.expensegroup_id
+    join workspaces w on w.id = eg.workspace_id
+    where e.org_id is null
+)
+update expenses e
+set org_id = eg.fyle_org_id
+from expense_groups eg
+where e.id = eg.id;

sql/scripts/023-fill-skipped-accounting-export-summary.sql

@@ -0,0 +1,10 @@
+rollback;
+begin;
+
+update expenses set accounting_export_summary = jsonb_build_object(
+    'id', expense_id,
+    'url', CONCAT('https://netsuite.fyleapps.tech/workspaces/', workspace_id, '/expense_groups?page_number=0&page_size=10&state=SKIP'),
+    'state', 'SKIPPED',
+    'synced', false,
+    'error_type', null
+) where is_skipped = 't';