diff --git a/apps/fyle/actions.py b/apps/fyle/actions.py index a1948006..5430bea8 100644 --- a/apps/fyle/actions.py +++ b/apps/fyle/actions.py @@ -1,22 +1,20 @@ -from typing import List import logging from datetime import datetime, timezone +from typing import List -from django.db.models import Q from django.conf import settings +from django.db.models import Q +from fyle.platform.exceptions import InternalServerError, RetryException +from fyle.platform.internals.decorators import retry from fyle_accounting_mappings.models import ExpenseAttribute from fyle_integrations_platform_connector import PlatformConnector +from apps.fyle.enums import FundSourceEnum, FyleAttributeEnum +from apps.fyle.helpers import get_batched_expenses, get_updated_accounting_export_summary +from apps.fyle.models import Expense, ExpenseGroup from apps.workspaces.models import FyleCredential, Workspace, WorkspaceGeneralSettings -from apps.fyle.enums import FyleAttributeEnum, FundSourceEnum -from apps.fyle.models import ExpenseGroup, Expense -from fyle.platform.internals.decorators import retry -from fyle.platform.exceptions import InternalServerError, RetryException -from apps.fyle.helpers import get_updated_accounting_export_summary, get_batched_expenses - - logger = logging.getLogger(__name__) logger.level = logging.INFO diff --git a/apps/fyle/helpers.py b/apps/fyle/helpers.py index 92c1c818..dc8b845c 100644 --- a/apps/fyle/helpers.py +++ b/apps/fyle/helpers.py @@ -1,11 +1,12 @@ import json -import traceback import logging +import traceback from typing import List, Union import requests from django.conf import settings -from apps.fyle.models import ExpenseGroupSettings, Expense + +from apps.fyle.models import Expense, ExpenseGroupSettings from apps.tasks.models import TaskLog from apps.workspaces.models import WorkspaceGeneralSettings diff --git a/apps/fyle/models.py b/apps/fyle/models.py index 9e1cdd31..b15820e2 100644 --- a/apps/fyle/models.py +++ b/apps/fyle/models.py @@ -13,11 +13,9 @@ from django.db.models import Count, JSONField from fyle_accounting_mappings.models import ExpenseAttribute +from apps.fyle.enums import ExpenseStateEnum, FundSourceEnum, PlatformExpensesEnum from apps.workspaces.models import Workspace -from .enums import FundSourceEnum, PlatformExpensesEnum, ExpenseStateEnum - - logger = logging.getLogger(__name__) logger.level = logging.INFO diff --git a/apps/fyle/tasks.py b/apps/fyle/tasks.py index d573c81d..267f4047 100644 --- a/apps/fyle/tasks.py +++ b/apps/fyle/tasks.py @@ -1,23 +1,21 @@ import logging import traceback from datetime import datetime -from typing import List, Dict +from typing import Dict, List from django.db import transaction from fyle.platform.exceptions import InvalidTokenError as FyleInvalidTokenError from fyle_integrations_platform_connector import PlatformConnector -from apps.tasks.models import TaskLog -from apps.tasks.enums import TaskLogStatusEnum, TaskLogTypeEnum -from apps.workspaces.models import FyleCredential, Workspace, WorkspaceGeneralSettings - +from apps.fyle.actions import create_generator_and_post_in_batches +from apps.fyle.enums import ExpenseStateEnum, FundSourceEnum, PlatformExpensesEnum +from apps.fyle.helpers import get_filter_credit_expenses, get_fund_source, get_source_account_type, handle_import_exception from apps.fyle.models import Expense, ExpenseGroup, ExpenseGroupSettings -from apps.fyle.enums import FundSourceEnum, PlatformExpensesEnum, ExpenseStateEnum -from apps.fyle.helpers import get_filter_credit_expenses, get_source_account_type, get_fund_source, handle_import_exception -from apps.workspaces.actions import export_to_xero from apps.fyle.queue import async_post_accounting_export_summary -from apps.fyle.actions import create_generator_and_post_in_batches - +from apps.tasks.enums import TaskLogStatusEnum, TaskLogTypeEnum +from apps.tasks.models import TaskLog +from apps.workspaces.actions import export_to_xero +from apps.workspaces.models import FyleCredential, Workspace, WorkspaceGeneralSettings logger = logging.getLogger(__name__) logger.level = logging.INFO diff --git a/apps/fyle/urls.py b/apps/fyle/urls.py index 153dc610..977c1732 100644 --- a/apps/fyle/urls.py +++ b/apps/fyle/urls.py @@ -6,9 +6,9 @@ ExpenseGroupSyncView, ExpenseGroupView, ExportableExpenseGroupsView, + ExportView, RefreshFyleDimensionView, SyncFyleDimensionView, - ExportView ) urlpatterns = [ diff --git a/apps/fyle/views.py b/apps/fyle/views.py index b00076cd..213b0711 100644 --- a/apps/fyle/views.py +++ b/apps/fyle/views.py @@ -5,10 +5,10 @@ from apps.exceptions import handle_view_exceptions from apps.fyle.actions import exportable_expense_group, get_expense_field, refresh_fyle_dimension, sync_fyle_dimension from apps.fyle.models import ExpenseGroup, ExpenseGroupSettings +from apps.fyle.queue import async_import_and_export_expenses from apps.fyle.serializers import ExpenseFieldSerializer, ExpenseGroupSerializer, ExpenseGroupSettingsSerializer from apps.fyle.tasks import async_create_expense_groups, get_task_log_and_fund_source from fyle_xero_api.utils import LookupFieldMixin -from apps.fyle.queue import async_import_and_export_expenses class ExpenseGroupView(LookupFieldMixin, generics.ListCreateAPIView): diff --git a/apps/mappings/helpers.py b/apps/mappings/helpers.py index dce2a16b..c33824ac 100644 --- a/apps/mappings/helpers.py +++ b/apps/mappings/helpers.py @@ -3,9 +3,8 @@ from django_q.models import Schedule from fyle_accounting_mappings.models import MappingSetting -from apps.workspaces.models import WorkspaceGeneralSettings - from apps.fyle.enums import FyleAttributeEnum +from apps.workspaces.models import WorkspaceGeneralSettings def schedule_or_delete_fyle_import_tasks(configuration: WorkspaceGeneralSettings): diff --git a/apps/mappings/signals.py b/apps/mappings/signals.py index 679ac494..d54b6313 100644 --- a/apps/mappings/signals.py +++ b/apps/mappings/signals.py @@ -8,13 +8,13 @@ from django_q.tasks import async_task from fyle_accounting_mappings.models import Mapping, MappingSetting +from apps.fyle.enums import FyleAttributeEnum from apps.mappings.helpers import schedule_or_delete_fyle_import_tasks from apps.mappings.models import TenantMapping from apps.mappings.queue import schedule_cost_centers_creation, schedule_fyle_attributes_creation from apps.mappings.tasks import upload_attributes_to_fyle from apps.tasks.models import Error from apps.workspaces.models import WorkspaceGeneralSettings -from apps.fyle.enums import FyleAttributeEnum @receiver(post_save, sender=Mapping) diff --git a/apps/mappings/tasks.py b/apps/mappings/tasks.py index 2893af7c..fd98da31 100644 --- a/apps/mappings/tasks.py +++ b/apps/mappings/tasks.py @@ -5,13 +5,12 @@ from fyle_accounting_mappings.models import DestinationAttribute, ExpenseAttribute, Mapping, MappingSetting from fyle_integrations_platform_connector import PlatformConnector +from apps.fyle.enums import FyleAttributeEnum from apps.mappings.constants import FYLE_EXPENSE_SYSTEM_FIELDS from apps.mappings.exceptions import handle_import_exceptions from apps.tasks.models import Error from apps.workspaces.models import FyleCredential, WorkspaceGeneralSettings, XeroCredentials from apps.xero.utils import XeroConnector -from apps.fyle.enums import FyleAttributeEnum - logger = logging.getLogger(__name__) logger.level = logging.INFO diff --git a/apps/tasks/models.py b/apps/tasks/models.py index e17e5c89..e68e1e85 100644 --- a/apps/tasks/models.py +++ b/apps/tasks/models.py @@ -3,11 +3,10 @@ from fyle_accounting_mappings.models import ExpenseAttribute from apps.fyle.models import ExpenseGroup +from apps.tasks.enums import ErrorTypeEnum from apps.workspaces.models import Workspace from apps.xero.models import BankTransaction, Bill, Payment -from .enums import ErrorTypeEnum - def get_default(): return dict diff --git a/apps/workspaces/actions.py b/apps/workspaces/actions.py index 236deda0..c2887511 100644 --- a/apps/workspaces/actions.py +++ b/apps/workspaces/actions.py @@ -9,16 +9,22 @@ from fyle_rest_auth.models import AuthToken from xerosdk import exceptions as xero_exc +from apps.fyle.enums import FundSourceEnum from apps.fyle.helpers import get_cluster_domain -from apps.fyle.models import ExpenseGroupSettings +from apps.fyle.models import ExpenseGroup, ExpenseGroupSettings from apps.mappings.models import TenantMapping -from apps.workspaces.models import FyleCredential, LastExportDetail, Workspace, XeroCredentials, WorkspaceGeneralSettings, WorkspaceSchedule -from apps.fyle.models import ExpenseGroup +from apps.workspaces.models import ( + FyleCredential, + LastExportDetail, + Workspace, + WorkspaceGeneralSettings, + WorkspaceSchedule, + XeroCredentials, +) from apps.workspaces.signals import post_delete_xero_connection from apps.workspaces.utils import generate_xero_refresh_token -from apps.xero.utils import XeroConnector -from apps.fyle.enums import FundSourceEnum from apps.xero.queue import schedule_bank_transaction_creation, schedule_bills_creation +from apps.xero.utils import XeroConnector logger = logging.getLogger(__name__) diff --git a/apps/workspaces/email.py b/apps/workspaces/email.py index a1f4c5c9..9caceca5 100644 --- a/apps/workspaces/email.py +++ b/apps/workspaces/email.py @@ -6,15 +6,11 @@ from django.template.loader import render_to_string from fyle_accounting_mappings.models import ExpenseAttribute from sendgrid import SendGridAPIClient -from sendgrid.helpers.mail import ( - Mail, From -) +from sendgrid.helpers.mail import From, Mail from apps.mappings.models import TenantMapping - +from apps.tasks.enums import TaskLogStatusEnum, TaskLogTypeEnum from apps.tasks.models import Error, TaskLog -from apps.tasks.enums import TaskLogTypeEnum, TaskLogStatusEnum - from apps.workspaces.models import Workspace, WorkspaceSchedule diff --git a/apps/workspaces/signals.py b/apps/workspaces/signals.py index dd1ef0b8..886262a6 100644 --- a/apps/workspaces/signals.py +++ b/apps/workspaces/signals.py @@ -1,10 +1,9 @@ from fyle_accounting_mappings.models import DestinationAttribute, Mapping +from apps.fyle.enums import FyleAttributeEnum from apps.mappings.models import TenantMapping from apps.workspaces.models import Workspace -from apps.fyle.enums import FyleAttributeEnum - def post_delete_xero_connection(workspace_id): """ diff --git a/apps/workspaces/tasks.py b/apps/workspaces/tasks.py index d4d2178a..aacfbae2 100644 --- a/apps/workspaces/tasks.py +++ b/apps/workspaces/tasks.py @@ -1,26 +1,21 @@ -import logging import json +import logging from datetime import datetime -from django.conf import settings +from django.conf import settings from fyle_integrations_platform_connector import PlatformConnector from fyle_rest_auth.helpers import get_fyle_admin -from apps.fyle.helpers import post_request -from apps.fyle.tasks import async_create_expense_groups from apps.fyle.enums import FundSourceEnum - +from apps.fyle.helpers import post_request +from apps.fyle.tasks import async_create_expense_groups from apps.mappings.models import TenantMapping - -from apps.tasks.models import TaskLog from apps.tasks.enums import TaskLogStatusEnum, TaskLogTypeEnum - +from apps.tasks.models import TaskLog from apps.users.models import User - +from apps.workspaces.actions import export_to_xero from apps.workspaces.email import get_admin_name, get_errors, get_failed_task_logs_count, send_failure_notification_email from apps.workspaces.models import FyleCredential, Workspace, WorkspaceGeneralSettings, WorkspaceSchedule -from apps.workspaces.actions import export_to_xero - logger = logging.getLogger(__name__) logger.level = logging.INFO diff --git a/apps/workspaces/utils.py b/apps/workspaces/utils.py index dc66ac42..7d28fe7b 100644 --- a/apps/workspaces/utils.py +++ b/apps/workspaces/utils.py @@ -10,15 +10,11 @@ from fyle_accounting_mappings.models import MappingSetting from xerosdk import InternalServerError, InvalidTokenError, XeroSDK -from apps.fyle.models import ExpenseGroupSettings from apps.fyle.enums import FyleAttributeEnum - +from apps.fyle.models import ExpenseGroupSettings from apps.mappings.queue import schedule_auto_map_employees, schedule_tax_groups_creation - from apps.workspaces.models import Workspace, WorkspaceGeneralSettings - from apps.xero.queue import schedule_payment_creation, schedule_reimbursements_sync, schedule_xero_objects_status_sync - from fyle_xero_api.utils import assert_valid diff --git a/apps/workspaces/views.py b/apps/workspaces/views.py index 048e6b17..00cd099e 100644 --- a/apps/workspaces/views.py +++ b/apps/workspaces/views.py @@ -1,16 +1,15 @@ import logging from django.contrib.auth import get_user_model +from django_q.tasks import async_task from fyle_rest_auth.utils import AuthUtils from rest_framework import generics from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.views import status -from django_q.tasks import async_task - from apps.exceptions import handle_view_exceptions -from apps.workspaces.actions import connect_xero, get_workspace_admin, post_workspace, revoke_connections +from apps.workspaces.actions import connect_xero, export_to_xero, get_workspace_admin, post_workspace, revoke_connections from apps.workspaces.models import LastExportDetail, Workspace, WorkspaceGeneralSettings, XeroCredentials from apps.workspaces.serializers import ( LastExportDetailSerializer, @@ -18,7 +17,6 @@ WorkspaceSerializer, XeroCredentialSerializer, ) -from apps.workspaces.actions import export_to_xero from apps.workspaces.utils import generate_xero_identity logger = logging.getLogger(__name__) diff --git a/apps/xero/actions.py b/apps/xero/actions.py index 7c604236..d6a7e13b 100644 --- a/apps/xero/actions.py +++ b/apps/xero/actions.py @@ -3,9 +3,9 @@ from django_q.tasks import Chain from fyle_accounting_mappings.models import MappingSetting +from apps.fyle.enums import FyleAttributeEnum from apps.workspaces.models import Workspace, WorkspaceGeneralSettings, XeroCredentials from apps.xero.utils import XeroConnector -from apps.fyle.enums import FyleAttributeEnum def get_xero_connector(workspace_id): diff --git a/apps/xero/exceptions.py b/apps/xero/exceptions.py index 50aa1f26..3d6c2c72 100644 --- a/apps/xero/exceptions.py +++ b/apps/xero/exceptions.py @@ -12,16 +12,12 @@ XeroSDKError, ) -from fyle_xero_api.exceptions import BulkError - +from apps.fyle.actions import update_failed_expenses from apps.fyle.models import ExpenseGroup - +from apps.tasks.enums import ErrorTypeEnum, TaskLogStatusEnum, TaskLogTypeEnum from apps.tasks.models import Error, TaskLog -from apps.tasks.enums import TaskLogStatusEnum, TaskLogTypeEnum, ErrorTypeEnum - from apps.workspaces.models import FyleCredential, LastExportDetail, XeroCredentials -from apps.fyle.actions import update_failed_expenses - +from fyle_xero_api.exceptions import BulkError logger = logging.getLogger(__name__) logger.level = logging.INFO diff --git a/apps/xero/models.py b/apps/xero/models.py index eda9f1ad..649b9a24 100644 --- a/apps/xero/models.py +++ b/apps/xero/models.py @@ -5,11 +5,9 @@ from django.db.models import JSONField from fyle_accounting_mappings.models import DestinationAttribute, ExpenseAttribute, Mapping, MappingSetting -from apps.fyle.models import Expense, ExpenseGroup from apps.fyle.enums import FyleAttributeEnum - +from apps.fyle.models import Expense, ExpenseGroup from apps.mappings.models import GeneralMapping - from apps.workspaces.models import FyleCredential, Workspace diff --git a/apps/xero/queue.py b/apps/xero/queue.py index 7b2ab416..7c25d42a 100644 --- a/apps/xero/queue.py +++ b/apps/xero/queue.py @@ -1,24 +1,16 @@ from datetime import datetime, timedelta - -from django_q.models import Schedule from typing import List from django.db.models import Q - +from django_q.models import Schedule from django_q.tasks import Chain - - from xerosdk.exceptions import UnsuccessfulAuthentication from apps.fyle.models import Expense, ExpenseGroup - from apps.mappings.models import GeneralMapping - -from apps.tasks.models import TaskLog from apps.tasks.enums import TaskLogStatusEnum, TaskLogTypeEnum - +from apps.tasks.models import TaskLog from apps.workspaces.models import FyleCredential, XeroCredentials - from apps.xero.utils import XeroConnector diff --git a/apps/xero/tasks.py b/apps/xero/tasks.py index 1deca1b4..6bce987c 100644 --- a/apps/xero/tasks.py +++ b/apps/xero/tasks.py @@ -5,31 +5,22 @@ from typing import List from django.db import transaction - from fyle_accounting_mappings.models import DestinationAttribute, ExpenseAttribute, Mapping - from fyle_integrations_platform_connector import PlatformConnector - -from fyle_xero_api.exceptions import BulkError - from xerosdk.exceptions import UnsuccessfulAuthentication, WrongParamsError -from apps.fyle.models import Expense, ExpenseGroup, Reimbursement +from apps.fyle.actions import update_complete_expenses, update_expenses_in_progress from apps.fyle.enums import FundSourceEnum, FyleAttributeEnum, PlatformExpensesEnum - +from apps.fyle.models import Expense, ExpenseGroup, Reimbursement +from apps.fyle.tasks import post_accounting_export_summary from apps.mappings.models import GeneralMapping, TenantMapping - +from apps.tasks.enums import ErrorTypeEnum, TaskLogStatusEnum, TaskLogTypeEnum from apps.tasks.models import Error, TaskLog -from apps.tasks.enums import TaskLogStatusEnum, TaskLogTypeEnum, ErrorTypeEnum - from apps.workspaces.models import FyleCredential, Workspace, WorkspaceGeneralSettings, XeroCredentials - from apps.xero.exceptions import handle_xero_exceptions from apps.xero.models import BankTransaction, BankTransactionLineItem, Bill, BillLineItem, Payment from apps.xero.utils import XeroConnector -from apps.fyle.actions import update_expenses_in_progress, update_complete_expenses -from apps.fyle.tasks import post_accounting_export_summary - +from fyle_xero_api.exceptions import BulkError logger = logging.getLogger(__name__) logger.level = logging.INFO diff --git a/scripts/python/create-admin-subscriptions.py b/scripts/python/create-admin-subscriptions.py index d462088d..5b0e79cf 100644 --- a/scripts/python/create-admin-subscriptions.py +++ b/scripts/python/create-admin-subscriptions.py @@ -1,9 +1,9 @@ # Create admin subscriptions for existing workspaces -from apps.workspaces.tasks import async_create_admin_subcriptions -from apps.workspaces.models import Workspace from fyle.platform.exceptions import InvalidTokenError as FyleInvalidTokenError +from apps.workspaces.models import Workspace +from apps.workspaces.tasks import async_create_admin_subcriptions workspaces = Workspace.objects.all() diff --git a/scripts/python/fill-accounting-export-summary.py b/scripts/python/fill-accounting-export-summary.py new file mode 100644 index 00000000..2b1ba2ec --- /dev/null +++ b/scripts/python/fill-accounting-export-summary.py @@ -0,0 +1,100 @@ +from datetime import datetime + +from django.conf import settings +from django.db.models import Q + +from apps.fyle.actions import __bulk_update_expenses +from apps.fyle.helpers import get_updated_accounting_export_summary +from apps.fyle.models import Expense +from apps.tasks.models import TaskLog +from apps.workspaces.models import Workspace + +# PLEASE RUN sql/scripts/022-fill-skipped-accounting-export-summary.sql BEFORE RUNNING THIS SCRIPT + + +export_types = ['CREATING_BILL', 'CREATING_BANK_TRANSACTION'] +task_statuses = ['COMPLETE', 'FAILED', 'FATAL'] + + +# We'll handle all COMPLETE, ERROR expenses in this script +workspaces = Workspace.objects.filter( + ~Q(name__icontains='fyle for') & ~Q(name__icontains='test') +) + +start_time = datetime.now() +number_of_expenses_without_accounting_export_summary = Expense.objects.filter( + accounting_export_summary__state__isnull=True +).count() +print('Number of expenses without accounting export summary - {}'.format(number_of_expenses_without_accounting_export_summary)) +for workspace in workspaces: + task_logs_count = TaskLog.objects.filter( + type__in=export_types, + workspace_id=workspace.id, + status__in=task_statuses + ).count() + print('Updating summary from workspace - {} with ID - {}'.format(workspace.name, workspace.id)) + print('Number of task logs to be updated - {}'.format(task_logs_count)) + page_size = 200 + for offset in range(0, task_logs_count, page_size): + expense_to_be_updated = [] + limit = offset + page_size + paginated_task_logs = TaskLog.objects.filter( + type__in=export_types, + workspace_id=workspace.id, + status__in=task_statuses + )[offset:limit] + for task_log in paginated_task_logs: + expense_group = task_log.expense_group + state = 'ERROR' if task_log.status == 'FAILED' or task_log.status == 'FATAL' else 'COMPLETE' + error_type = None + url = None + if task_log.status == 'FAILED' or task_log.status == 'FATAL': + for item in task_log.detail: + if item.get('type') and item.get('type') == 'Category Mapping': + error_type = 'MAPPING' + else: + error_type = 'ACCOUNTING_INTEGRATION_ERROR' + url = '{}/workspaces/main/dashboard'.format(settings.XERO_INTEGRATION_APP_URL) + else: + try: + if task_log.type == 'CREATING_BILL': + export_id = expense_group.response_logs['Invoices'][0]['InvoiceID'] + if workspace.xero_short_code: + url = f'https://go.xero.com/organisationlogin/default.aspx?shortcode={workspace.xero_short_code}&redirecturl=/AccountsPayable/Edit.aspx?InvoiceID={export_id}' + else: + url = f'https://go.xero.com/AccountsPayable/View.aspx?invoiceID={export_id}' + else: + export_id = expense_group.response_logs['BankTransactions'][0]['BankTransactionID'] + account_id = expense_group.response_logs['BankTransactions'][0]['BankAccount']['AccountID'] + if workspace.xero_short_code: + url = f'https://go.xero.com/organisationlogin/default.aspx?shortcode={workspace.xero_short_code}&redirecturl=/Bank/ViewTransaction.aspx?bankTransactionID={export_id}&accountID={account_id}' + else: + url = f'https://go.xero.com/Bank/ViewTransaction.aspx?bankTransactionID={export_id}&accountID={account_id}' + except Exception as error: + # Defaulting it to Intacct app url, worst case scenario if we're not able to parse it properly + url = 'https://go.xero.com' + print('Error while parsing url for task log - {}. Error - {}'.format(task_log.id, error)) + for expense in expense_group.expenses.filter(accounting_export_summary__state__isnull=True): + if url: + expense_to_be_updated.append( + Expense( + id=expense.id, + accounting_export_summary=get_updated_accounting_export_summary( + expense.expense_id, + state, + error_type, + url, + False + ) + ) + ) + print('Updating {} expenses in batches of 50'.format(len(expense_to_be_updated))) + __bulk_update_expenses(expense_to_be_updated) + + +number_of_expenses_without_accounting_export_summary = Expense.objects.filter( + accounting_export_summary__state__isnull=True +).count() +print('Number of expenses without accounting export summary - {}'.format(number_of_expenses_without_accounting_export_summary)) +end_time = datetime.now() +print('Time taken - {}'.format(end_time - start_time)) diff --git a/scripts/python/post-accounting-export-summary.py b/scripts/python/post-accounting-export-summary.py new file mode 100644 index 00000000..4713fc7c --- /dev/null +++ b/scripts/python/post-accounting-export-summary.py @@ -0,0 +1,43 @@ +from datetime import datetime +from time import sleep + +from django.db.models import Q + +from apps.fyle.models import Expense +from apps.fyle.tasks import post_accounting_export_summary +from apps.workspaces.models import Workspace + +# PLEASE RUN scripts/python/fill-accounting-export-summary.py BEFORE RUNNING THIS SCRIPT +workspaces = Workspace.objects.filter( + ~Q(name__icontains='fyle for') & ~Q(name__icontains='test') +) + +start_time = datetime.now() +number_of_expenses_to_be_posted = Expense.objects.filter( + accounting_export_summary__synced=False +).count() +print('Number of expenses to be posted - {}'.format(number_of_expenses_to_be_posted)) +for workspace in workspaces: + expenses_count = Expense.objects.filter( + accounting_export_summary__synced=False, + workspace_id=workspace.id + ).count() + print('Updating summary from workspace - {} with ID - {}'.format(workspace.name, workspace.id)) + print('Number of expenses_count to be posted for the current workspace - {}'.format(expenses_count)) + if expenses_count: + try: + sleep(1) + post_accounting_export_summary(workspace.fyle_org_id, workspace.id) + except Exception as e: + print('Error while posting accounting export summary for workspace - {} with ID - {}'.format(workspace.name, workspace.id)) + print(e.__dict__) + +number_of_expenses_posted = Expense.objects.filter( + accounting_export_summary__synced=True +).count() +print('Number of expenses posted to Fyle - {}'.format(number_of_expenses_posted)) +end_time = datetime.now() +print('Time taken - {}'.format(end_time - start_time)) + +# This query should return 0 rows +# select expense_group_id from task_logs where status not in ('ENQUEUED', 'IN_PROGRESS') and expense_group_id in (select expensegroup_id from expense_groups_expenses where expense_id in (select id from expenses where accounting_export_summary ='{}')); diff --git a/scripts/python/post-integrations.py b/scripts/python/post-integrations.py index c33812f2..43c8292a 100644 --- a/scripts/python/post-integrations.py +++ b/scripts/python/post-integrations.py @@ -1,5 +1,5 @@ -from apps.workspaces.tasks import post_to_integration_settings from apps.workspaces.models import Workspace +from apps.workspaces.tasks import post_to_integration_settings workspaces = Workspace.objects.filter(onboarding_state='COMPLETE') diff --git a/sql/scripts/022-fill-org-id-expenses.sql b/sql/scripts/022-fill-org-id-expenses.sql new file mode 100644 index 00000000..1df08fd6 --- /dev/null +++ b/sql/scripts/022-fill-org-id-expenses.sql @@ -0,0 +1,14 @@ +rollback; +begin; + +with expense_groups as ( + select w.fyle_org_id, e.id from expenses e + join expense_groups_expenses ege on e.id = ege.expense_id + join expense_groups eg on eg.id = ege.expensegroup_id + join workspaces w on w.id = eg.workspace_id + where e.org_id is null +) +update expenses e +set org_id = eg.fyle_org_id +from expense_groups eg +where e.id = eg.id; diff --git a/sql/scripts/023-fill-skipped-accounting-export-summary.sql b/sql/scripts/023-fill-skipped-accounting-export-summary.sql new file mode 100644 index 00000000..92672074 --- /dev/null +++ b/sql/scripts/023-fill-skipped-accounting-export-summary.sql @@ -0,0 +1,10 @@ +rollback; +begin; + +update expenses set accounting_export_summary = jsonb_build_object( + 'id', expense_id, + 'url', CONCAT('https://netsuite.fyleapps.com/workspaces/', workspace_id, '/expense_groups?page_number=0&page_size=10&state=SKIP'), + 'state', 'SKIPPED', + 'synced', false, + 'error_type', null +) where is_skipped = 't'; diff --git a/start_import_qcluster.sh b/start_import_qcluster.sh index 5bc0ffd8..cfad5e4c 100644 --- a/start_import_qcluster.sh +++ b/start_import_qcluster.sh @@ -3,4 +3,4 @@ # Creating the cache table python manage.py createcachetable --database cache_db -Q_CLUSTER_NAME=import python manage.py qcluster \ No newline at end of file +Q_CLUSTER_NAME=import python manage.py qcluster diff --git a/tests/sql_fixtures/reset_db_fixtures/reset_db.sql b/tests/sql_fixtures/reset_db_fixtures/reset_db.sql index c6f7bb50..0c05291e 100644 --- a/tests/sql_fixtures/reset_db_fixtures/reset_db.sql +++ b/tests/sql_fixtures/reset_db_fixtures/reset_db.sql @@ -6589,4 +6589,3 @@ ALTER TABLE ONLY public.xero_credentials -- -- PostgreSQL database dump complete -- - diff --git a/tests/test_fyle/test_actions.py b/tests/test_fyle/test_actions.py index 44cf778d..65a216dd 100644 --- a/tests/test_fyle/test_actions.py +++ b/tests/test_fyle/test_actions.py @@ -1,21 +1,20 @@ from unittest import mock -from django.conf import settings -from django.db.models import Q -from fyle_integrations_platform_connector import PlatformConnector +from django.conf import settings from fyle.platform.exceptions import InternalServerError, RetryException, WrongParamsError +from fyle_integrations_platform_connector import PlatformConnector -from apps.fyle.models import Expense, ExpenseGroup -from apps.workspaces.models import Workspace, FyleCredential from apps.fyle.actions import ( - update_expenses_in_progress, + bulk_post_accounting_export_summary, create_generator_and_post_in_batches, mark_accounting_export_summary_as_synced, update_complete_expenses, - bulk_post_accounting_export_summary, - update_failed_expenses + update_expenses_in_progress, + update_failed_expenses, ) from apps.fyle.helpers import get_updated_accounting_export_summary +from apps.fyle.models import Expense +from apps.workspaces.models import FyleCredential def test_update_expenses_in_progress(db): @@ -33,6 +32,7 @@ def test_update_expenses_in_progress(db): assert expense.accounting_export_summary['error_type'] == None assert expense.accounting_export_summary['id'] == expense.expense_id + def test_update_failed_expenses(db): expenses = Expense.objects.filter(org_id='or79Cob97KSh') update_failed_expenses(expenses, True) @@ -48,6 +48,7 @@ def test_update_failed_expenses(db): ) assert expense.accounting_export_summary['id'] == expense.expense_id + def test_update_complete_expenses(db): expenses = Expense.objects.filter(org_id='or79Cob97KSh') @@ -135,6 +136,7 @@ def test_mark_accounting_export_summary_as_synced(db): for expense in expenses: assert expense.accounting_export_summary['synced'] == True + def test_bulk_post_accounting_export_summary(db): fyle_credentails = FyleCredential.objects.get(workspace_id=1) platform = PlatformConnector(fyle_credentails) diff --git a/tests/test_fyle/test_helpers.py b/tests/test_fyle/test_helpers.py index 59d06d7d..e7be57fc 100644 --- a/tests/test_fyle/test_helpers.py +++ b/tests/test_fyle/test_helpers.py @@ -1,12 +1,12 @@ from asyncio.log import logger -from django.conf import settings +from django.conf import settings from rest_framework.response import Response from rest_framework.views import status -from apps.fyle.helpers import get_fyle_orgs, get_request, post_request, get_updated_accounting_export_summary -from apps.fyle.models import Expense from apps.fyle.actions import __bulk_update_expenses +from apps.fyle.helpers import get_fyle_orgs, get_request, get_updated_accounting_export_summary, post_request +from apps.fyle.models import Expense def test_post_request(mocker): diff --git a/tests/test_fyle/test_queue.py b/tests/test_fyle/test_queue.py index 86ddec9e..81a2ea24 100644 --- a/tests/test_fyle/test_queue.py +++ b/tests/test_fyle/test_queue.py @@ -1,8 +1,7 @@ from apps.fyle.models import Expense +from apps.fyle.queue import async_import_and_export_expenses, async_post_accounting_export_summary from apps.workspaces.models import FyleCredential, XeroCredentials from apps.xero.queue import __create_chain_and_run -from apps.fyle.queue import async_post_accounting_export_summary, async_import_and_export_expenses - from apps.xero.utils import XeroConnector @@ -11,6 +10,7 @@ def test_async_post_accounting_export_summary(db): async_post_accounting_export_summary(1, 1) assert True + # This test is just for cov :D def test_create_chain_and_run(db): workspace_id = 1 diff --git a/tests/test_fyle/test_task.py b/tests/test_fyle/test_task.py index b11336e7..ff35f1f0 100644 --- a/tests/test_fyle/test_task.py +++ b/tests/test_fyle/test_task.py @@ -2,13 +2,12 @@ from fyle.platform.exceptions import InvalidTokenError as FyleInvalidTokenError -from apps.fyle.models import ExpenseGroupSettings, Expense, ExpenseGroup -from django.db.models import Q +from apps.fyle.actions import update_expenses_in_progress +from apps.fyle.models import Expense, ExpenseGroup, ExpenseGroupSettings +from apps.fyle.tasks import create_expense_groups, post_accounting_export_summary from apps.tasks.models import TaskLog -from apps.workspaces.models import FyleCredential, Workspace +from apps.workspaces.models import FyleCredential from tests.test_fyle.fixtures import data -from apps.fyle.tasks import create_expense_groups, post_accounting_export_summary -from apps.fyle.actions import update_expenses_in_progress def test_create_expense_groups(mocker, db): @@ -69,8 +68,6 @@ def test_post_accounting_export_summary(db, mocker): expense = expense_group.expenses.first() expense_group.expenses.remove(expense.id) - workspace = Workspace.objects.get(id=1) - expense = Expense.objects.filter(id=expense.id).first() expense.workspace_id = 1 expense.save() diff --git a/tests/test_workspaces/test_apis/test_advanced_settings/test_views.py b/tests/test_workspaces/test_apis/test_advanced_settings/test_views.py index cab00c76..3f0391f0 100644 --- a/tests/test_workspaces/test_apis/test_advanced_settings/test_views.py +++ b/tests/test_workspaces/test_apis/test_advanced_settings/test_views.py @@ -1,6 +1,6 @@ import json -from apps.workspaces.models import Workspace, WorkspaceGeneralSettings, FyleCredential +from apps.workspaces.models import FyleCredential, Workspace, WorkspaceGeneralSettings from tests.helper import dict_compare_keys from tests.test_workspaces.test_apis.test_advanced_settings.fixtures import data diff --git a/tests/test_workspaces/test_tasks.py b/tests/test_workspaces/test_tasks.py index dfaa4a5c..85773779 100644 --- a/tests/test_workspaces/test_tasks.py +++ b/tests/test_workspaces/test_tasks.py @@ -9,12 +9,12 @@ from apps.workspaces.queue import schedule_sync from apps.workspaces.tasks import ( async_add_admins_to_workspace, + async_create_admin_subcriptions, async_update_fyle_credentials, async_update_workspace_name, + post_to_integration_settings, run_email_notification, run_sync_schedule, - async_create_admin_subcriptions, - post_to_integration_settings ) from tests.test_fyle.fixtures import data as fyle_data from tests.test_workspaces.fixtures import data diff --git a/tests/test_xero/test_tasks.py b/tests/test_xero/test_tasks.py index 3bd945d8..293b0864 100644 --- a/tests/test_xero/test_tasks.py +++ b/tests/test_xero/test_tasks.py @@ -13,7 +13,13 @@ from apps.workspaces.models import LastExportDetail, WorkspaceGeneralSettings, XeroCredentials from apps.xero.exceptions import update_last_export_details from apps.xero.models import BankTransaction, BankTransactionLineItem, Bill, BillLineItem -from apps.xero.queue import schedule_payment_creation, schedule_reimbursements_sync, schedule_xero_objects_status_sync, schedule_bank_transaction_creation, schedule_bills_creation +from apps.xero.queue import ( + schedule_bank_transaction_creation, + schedule_bills_creation, + schedule_payment_creation, + schedule_reimbursements_sync, + schedule_xero_objects_status_sync, +) from apps.xero.tasks import ( __validate_expense_group, attach_customer_to_export,