From 952a06abfd63be804ffbb6e46430ee93d5a53387 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Tue, 24 Sep 2024 10:09:11 +0530 Subject: [PATCH 1/2] feat: Add Receipt URLs to DB --- apps/netsuite/models.py | 8 ++++++++ apps/netsuite/tasks.py | 26 ++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/apps/netsuite/models.py b/apps/netsuite/models.py index 104038a8..ca14061b 100644 --- a/apps/netsuite/models.py +++ b/apps/netsuite/models.py @@ -378,6 +378,8 @@ class Bill(models.Model): is_retired = models.BooleanField(help_text='Is Payment sync retried', default=False) created_at = models.DateTimeField(auto_now_add=True, help_text='Created at') updated_at = models.DateTimeField(auto_now=True, help_text='Updated at') + receipt_urls = JSONField(help_text='Expense Receipt URL Map', null=True) + is_attachment_upload_failed = models.BooleanField(help_text='Is Attachment Upload Failed', default=False) class Meta: db_table = 'bills' @@ -580,6 +582,8 @@ class CreditCardCharge(models.Model): transaction_date = models.DateTimeField(help_text='CC Charge transaction date') created_at = models.DateTimeField(auto_now_add=True, help_text='Created at') updated_at = models.DateTimeField(auto_now=True, help_text='Updated at') + receipt_urls = JSONField(help_text='Expense Receipt URL Map', null=True) + is_attachment_upload_failed = models.BooleanField(help_text='Is Attachment Upload Failed', default=False) class Meta: db_table = 'credit_card_charges' @@ -796,6 +800,8 @@ class ExpenseReport(models.Model): is_retired = models.BooleanField(help_text='Is Payment sync retried', default=False) created_at = models.DateTimeField(auto_now_add=True, help_text='Created at') updated_at = models.DateTimeField(auto_now=True, help_text='Updated at') + receipt_urls = JSONField(help_text='Expense Receipt URL Map', null=True) + is_attachment_upload_failed = models.BooleanField(help_text='Is Attachment Upload Failed', default=False) class Meta: db_table = 'expense_reports' @@ -1025,6 +1031,8 @@ class JournalEntry(models.Model): paid_on_netsuite = models.BooleanField(help_text='Payment Status in NetSuite', default=False) created_at = models.DateTimeField(auto_now_add=True, help_text='Created at') updated_at = models.DateTimeField(auto_now=True, help_text='Updated at') + receipt_urls = JSONField(help_text='Expense Receipt URL Map', null=True) + is_attachment_upload_failed = models.BooleanField(help_text='Is Attachment Upload Failed', default=False) class Meta: db_table = 'journal_entries' diff --git a/apps/netsuite/tasks.py b/apps/netsuite/tasks.py index 0115083c..76818ae1 100644 --- a/apps/netsuite/tasks.py +++ b/apps/netsuite/tasks.py @@ -50,6 +50,12 @@ 'CREATING_CREDIT_CARD_CHARGE': 'construct_credit_card_charge_lineitems' } +TASK_TYPE_MODEL_MAP = { + 'CREATING_EXPENSE_REPORT': ExpenseReport, + 'CREATING_BILL': Bill, + 'CREATING_JOURNAL_ENTRY': JournalEntry +} + TASK_TYPE_EXPORT_COL_MAP = { 'CREATING_EXPENSE_REPORT': 'expense_report', 'CREATING_BILL': 'bill', @@ -90,7 +96,7 @@ def update_expense_and_post_summary(in_progress_expenses: List[Expense], workspa post_accounting_export_summary(fyle_org_id, workspace_id, fund_source) -def load_attachments(netsuite_connection: NetSuiteConnector, expense: Expense, expense_group: ExpenseGroup): +def load_attachments(netsuite_connection: NetSuiteConnector, expense: Expense, expense_group: ExpenseGroup, credit_card_charge_object: CreditCardCharge): """ Get attachments from Fyle :param netsuite_connection: NetSuite Connection @@ -145,6 +151,8 @@ def load_attachments(netsuite_connection: NetSuiteConnector, expense: Expense, e except InvalidTokenError: logger.info('Invalid Fyle refresh token for workspace %s', workspace_id) + credit_card_charge_object.is_attachment_upload_failed = True + credit_card_charge_object.save() except Exception: error = traceback.format_exc() @@ -152,6 +160,8 @@ def load_attachments(netsuite_connection: NetSuiteConnector, expense: Expense, e 'Attachment failed for expense group id %s / workspace id %s Error: %s', expense.expense_id, workspace_id, {'error': error} ) + credit_card_charge_object.is_attachment_upload_failed = True + credit_card_charge_object.save() def get_or_create_credit_card_vendor(expense_group: ExpenseGroup, merchant: str, auto_create_merchants: bool): @@ -368,6 +378,9 @@ def upload_attachments_and_update_export(expenses: List[Expense], task_log: Task netsuite_connection = NetSuiteConnector(netsuite_credentials, workspace_id) workspace = netsuite_credentials.workspace + task_model = TASK_TYPE_MODEL_MAP[task_log.type] + task_model = task_model.objects.get(expense_group_id=task_log.expense_group_id) + platform = PlatformConnector(fyle_credentials=fyle_credentials) expense_id_receipt_url_map = {} @@ -408,15 +421,21 @@ def upload_attachments_and_update_export(expenses: List[Expense], task_log: Task expense_id_receipt_url_map[expense.expense_id] = receipt_url construct_payload_and_update_export(expense_id_receipt_url_map, task_log, workspace, fyle_credentials.cluster_domain, netsuite_connection) + task_model.receipt_urls = json.dumps(expense_id_receipt_url_map) + task_model.save() except (NetSuiteRateLimitError, NetSuiteRequestError, NetSuiteLoginError, InvalidTokenError) as exception: logger.info('Error while uploading attachments to netsuite workspace_id - %s %s', workspace_id, exception.__dict__) + task_model.is_attachment_upload_failed = True + task_model.save() except Exception as exception: logger.error( 'Error while uploading attachments to netsuite workspace_id - %s %s %s', workspace_id, exception, traceback.format_exc() ) + task_model.is_attachment_upload_failed = True + task_model.save() def resolve_errors_for_exported_expense_group(expense_group, workspace_id=None): @@ -539,11 +558,14 @@ def create_credit_card_charge(expense_group, task_log_id, last_export): if expense.amount < 0: refund = True - attachment_link = load_attachments(netsuite_connection, expense, expense_group) + attachment_link = load_attachments(netsuite_connection, expense, expense_group, credit_card_charge_object) if attachment_link: attachment_links[expense.expense_id] = attachment_link + credit_card_charge_object.receipt_urls = json.dumps(attachment_links) + credit_card_charge_object.save() + created_credit_card_charge = netsuite_connection.post_credit_card_charge( credit_card_charge_object, credit_card_charge_lineitems_object, attachment_links, refund ) From e94720e75cc4eb950ac4e78febd84784a528bad9 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Tue, 24 Sep 2024 12:23:54 +0530 Subject: [PATCH 2/2] Fix tests and comments --- .../migrations/0027_auto_20240924_0614.py | 53 +++++++++++++++++++ apps/netsuite/tasks.py | 6 +-- .../reset_db_fixtures/reset_db.sql | 27 ++++++---- tests/test_netsuite/test_tasks.py | 18 +++++-- 4 files changed, 88 insertions(+), 16 deletions(-) create mode 100644 apps/netsuite/migrations/0027_auto_20240924_0614.py diff --git a/apps/netsuite/migrations/0027_auto_20240924_0614.py b/apps/netsuite/migrations/0027_auto_20240924_0614.py new file mode 100644 index 00000000..366137c9 --- /dev/null +++ b/apps/netsuite/migrations/0027_auto_20240924_0614.py @@ -0,0 +1,53 @@ +# Generated by Django 3.2.14 on 2024-09-24 06:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('netsuite', '0026_auto_20240902_1650'), + ] + + operations = [ + migrations.AddField( + model_name='bill', + name='is_attachment_upload_failed', + field=models.BooleanField(default=False, help_text='Is Attachment Upload Failed'), + ), + migrations.AddField( + model_name='bill', + name='receipt_urls', + field=models.JSONField(help_text='Expense Receipt URL Map', null=True), + ), + migrations.AddField( + model_name='creditcardcharge', + name='is_attachment_upload_failed', + field=models.BooleanField(default=False, help_text='Is Attachment Upload Failed'), + ), + migrations.AddField( + model_name='creditcardcharge', + name='receipt_urls', + field=models.JSONField(help_text='Expense Receipt URL Map', null=True), + ), + migrations.AddField( + model_name='expensereport', + name='is_attachment_upload_failed', + field=models.BooleanField(default=False, help_text='Is Attachment Upload Failed'), + ), + migrations.AddField( + model_name='expensereport', + name='receipt_urls', + field=models.JSONField(help_text='Expense Receipt URL Map', null=True), + ), + migrations.AddField( + model_name='journalentry', + name='is_attachment_upload_failed', + field=models.BooleanField(default=False, help_text='Is Attachment Upload Failed'), + ), + migrations.AddField( + model_name='journalentry', + name='receipt_urls', + field=models.JSONField(help_text='Expense Receipt URL Map', null=True), + ), + ] diff --git a/apps/netsuite/tasks.py b/apps/netsuite/tasks.py index 76818ae1..a457f6d8 100644 --- a/apps/netsuite/tasks.py +++ b/apps/netsuite/tasks.py @@ -379,7 +379,7 @@ def upload_attachments_and_update_export(expenses: List[Expense], task_log: Task workspace = netsuite_credentials.workspace task_model = TASK_TYPE_MODEL_MAP[task_log.type] - task_model = task_model.objects.get(expense_group_id=task_log.expense_group_id) + task_model = task_model.objects.get(expense_group_id=task_log.expense_group_id, expense_group__workspace_id=workspace_id) platform = PlatformConnector(fyle_credentials=fyle_credentials) @@ -421,7 +421,7 @@ def upload_attachments_and_update_export(expenses: List[Expense], task_log: Task expense_id_receipt_url_map[expense.expense_id] = receipt_url construct_payload_and_update_export(expense_id_receipt_url_map, task_log, workspace, fyle_credentials.cluster_domain, netsuite_connection) - task_model.receipt_urls = json.dumps(expense_id_receipt_url_map) + task_model.receipt_urls = expense_id_receipt_url_map task_model.save() except (NetSuiteRateLimitError, NetSuiteRequestError, NetSuiteLoginError, InvalidTokenError) as exception: @@ -563,7 +563,7 @@ def create_credit_card_charge(expense_group, task_log_id, last_export): if attachment_link: attachment_links[expense.expense_id] = attachment_link - credit_card_charge_object.receipt_urls = json.dumps(attachment_links) + credit_card_charge_object.receipt_urls = attachment_links credit_card_charge_object.save() created_credit_card_charge = netsuite_connection.post_credit_card_charge( diff --git a/tests/sql_fixtures/reset_db_fixtures/reset_db.sql b/tests/sql_fixtures/reset_db_fixtures/reset_db.sql index d1cfd160..1770a0ef 100644 --- a/tests/sql_fixtures/reset_db_fixtures/reset_db.sql +++ b/tests/sql_fixtures/reset_db_fixtures/reset_db.sql @@ -225,7 +225,9 @@ CREATE TABLE public.bills ( department_id character varying(255), override_tax_details boolean NOT NULL, class_id character varying(255), - is_retired boolean NOT NULL + is_retired boolean NOT NULL, + is_attachment_upload_failed boolean NOT NULL, + receipt_urls jsonb ); @@ -394,7 +396,9 @@ CREATE TABLE public.credit_card_charges ( expense_group_id integer NOT NULL, reference_number character varying(255), department_id character varying(255), - class_id character varying(255) + class_id character varying(255), + is_attachment_upload_failed boolean NOT NULL, + receipt_urls jsonb ); @@ -1066,7 +1070,9 @@ CREATE TABLE public.expense_reports ( payment_synced boolean NOT NULL, paid_on_netsuite boolean NOT NULL, credit_card_account_id character varying(255), - is_retired boolean NOT NULL + is_retired boolean NOT NULL, + is_attachment_upload_failed boolean NOT NULL, + receipt_urls jsonb ); @@ -1520,7 +1526,9 @@ CREATE TABLE public.journal_entries ( location_id character varying(255), payment_synced boolean NOT NULL, paid_on_netsuite boolean NOT NULL, - department_id character varying(255) + department_id character varying(255), + is_attachment_upload_failed boolean NOT NULL, + receipt_urls jsonb ); @@ -2664,7 +2672,7 @@ COPY public.bill_lineitems (id, account_id, location_id, department_id, class_id -- Data for Name: bills; Type: TABLE DATA; Schema: public; Owner: postgres -- -COPY public.bills (id, entity_id, accounts_payable_id, subsidiary_id, location_id, currency, memo, external_id, created_at, updated_at, expense_group_id, transaction_date, payment_synced, paid_on_netsuite, reference_number, department_id, override_tax_details, class_id, is_retired) FROM stdin; +COPY public.bills (id, entity_id, accounts_payable_id, subsidiary_id, location_id, currency, memo, external_id, created_at, updated_at, expense_group_id, transaction_date, payment_synced, paid_on_netsuite, reference_number, department_id, override_tax_details, class_id, is_retired, is_attachment_upload_failed, receipt_urls) FROM stdin; \. @@ -2704,7 +2712,7 @@ COPY public.credit_card_charge_lineitems (id, account_id, location_id, departmen -- Data for Name: credit_card_charges; Type: TABLE DATA; Schema: public; Owner: postgres -- -COPY public.credit_card_charges (id, credit_card_account_id, entity_id, subsidiary_id, location_id, currency, memo, external_id, transaction_date, created_at, updated_at, expense_group_id, reference_number, department_id, class_id) FROM stdin; +COPY public.credit_card_charges (id, credit_card_account_id, entity_id, subsidiary_id, location_id, currency, memo, external_id, transaction_date, created_at, updated_at, expense_group_id, reference_number, department_id, class_id, is_attachment_upload_failed, receipt_urls) FROM stdin; \. @@ -7976,6 +7984,7 @@ COPY public.django_migrations (id, app, name, applied) FROM stdin; 195 fyle 0032_alter_expensefilter_custom_field_type 2024-05-14 13:47:37.73285+00 196 fyle 0033_expense_paid_on_fyle 2024-06-18 16:52:33.560638+00 197 netsuite 0026_auto_20240902_1650 2024-09-02 16:50:55.770274+00 +198 netsuite 0027_auto_20240924_0614 2024-09-24 06:17:13.86751+00 \. @@ -11609,7 +11618,7 @@ COPY public.expense_report_lineitems (id, amount, category, class_id, customer_i -- Data for Name: expense_reports; Type: TABLE DATA; Schema: public; Owner: postgres -- -COPY public.expense_reports (id, account_id, entity_id, currency, department_id, class_id, location_id, subsidiary_id, memo, external_id, created_at, updated_at, expense_group_id, transaction_date, payment_synced, paid_on_netsuite, credit_card_account_id, is_retired) FROM stdin; +COPY public.expense_reports (id, account_id, entity_id, currency, department_id, class_id, location_id, subsidiary_id, memo, external_id, created_at, updated_at, expense_group_id, transaction_date, payment_synced, paid_on_netsuite, credit_card_account_id, is_retired, is_attachment_upload_failed, receipt_urls) FROM stdin; \. @@ -11660,7 +11669,7 @@ COPY public.import_logs (id, attribute_type, status, error_log, total_batches_co -- Data for Name: journal_entries; Type: TABLE DATA; Schema: public; Owner: postgres -- -COPY public.journal_entries (id, currency, subsidiary_id, memo, external_id, created_at, updated_at, expense_group_id, transaction_date, location_id, payment_synced, paid_on_netsuite, department_id) FROM stdin; +COPY public.journal_entries (id, currency, subsidiary_id, memo, external_id, created_at, updated_at, expense_group_id, transaction_date, location_id, payment_synced, paid_on_netsuite, department_id, is_attachment_upload_failed, receipt_urls) FROM stdin; \. @@ -11887,7 +11896,7 @@ SELECT pg_catalog.setval('public.django_content_type_id_seq', 47, true); -- Name: django_migrations_id_seq; Type: SEQUENCE SET; Schema: public; Owner: postgres -- -SELECT pg_catalog.setval('public.django_migrations_id_seq', 197, true); +SELECT pg_catalog.setval('public.django_migrations_id_seq', 198, true); -- diff --git a/tests/test_netsuite/test_tasks.py b/tests/test_netsuite/test_tasks.py index 8f618e73..96308763 100644 --- a/tests/test_netsuite/test_tasks.py +++ b/tests/test_netsuite/test_tasks.py @@ -635,6 +635,7 @@ def __init__(self, text, status_code): assert credit_card_charge.currency == '1' assert credit_card_charge.credit_card_account_id == '10' assert credit_card_charge.external_id == 'cc-charge 2 - ashwin.t@fyle.in' + assert credit_card_charge.receipt_urls == {'txy6folbrG2j': 'https://aaa.bbb.cc/x232sds'} task_log.status = 'READY' task_log.save() @@ -823,7 +824,7 @@ def test_check_netsuite_object_status_exception(create_bill_task, create_expense check_netsuite_object_status(1) -def test_load_attachments(db, add_netsuite_credentials, add_fyle_credentials, mocker): +def test_load_attachments(db, add_netsuite_credentials, add_fyle_credentials, create_credit_card_charge, mocker): mocker.patch( 'netsuitesdk.api.folders.Folders.post', return_value={'internalId': 'qwertyui', 'externalId': 'sdfghjk'} @@ -859,10 +860,12 @@ def test_load_attachments(db, add_netsuite_credentials, add_fyle_credentials, mo expense.file_ids = ['sdfghjk'] expense.save() + credit_card_charge_object = CreditCardCharge.objects.filter().first() + netsuite_credentials = NetSuiteCredentials.objects.get(workspace_id=1) netsuite_connection = NetSuiteConnector(netsuite_credentials, expense_group.workspace_id) - attachment = load_attachments(netsuite_connection, expense_group.expenses.first(), expense_group) + attachment = load_attachments(netsuite_connection, expense_group.expenses.first(), expense_group, credit_card_charge_object) assert attachment == 'https://aaa.bbb.cc/x232sds' mocker.patch( @@ -876,12 +879,13 @@ def test_load_attachments(db, add_netsuite_credentials, add_fyle_credentials, mo }] ) - attachment = load_attachments(netsuite_connection, expense_group.expenses.first(), expense_group) + attachment = load_attachments(netsuite_connection, expense_group.expenses.first(), expense_group, credit_card_charge_object) assert attachment == None fyle_credentials = FyleCredential.objects.get(workspace_id=1) fyle_credentials.delete() - attachment = load_attachments(netsuite_connection, expense_group.expenses.first(), expense_group) + attachment = load_attachments(netsuite_connection, expense_group.expenses.first(), expense_group, credit_card_charge_object) + assert credit_card_charge_object.is_attachment_upload_failed == True def test_create_or_update_employee_mapping(mocker, db): @@ -1621,6 +1625,8 @@ def test_upload_attachments_and_update_export(mocker, db): # asserting if the file is present lineitem = BillLineitem.objects.get(expense_id=1) assert lineitem.netsuite_receipt_url == 'https://aaa.bbb.cc/x232sds' + bill_object = Bill.objects.get(expense_group=expense_group) + assert bill_object.receipt_urls == {'txjvDntD9ZXR': 'https://aaa.bbb.cc/x232sds'} # mocking journal entry creation with the file being present @@ -1657,6 +1663,8 @@ def test_upload_attachments_and_update_export(mocker, db): # asserting if the file is present lineitem = JournalEntryLineItem.objects.get(expense_id=1) assert lineitem.netsuite_receipt_url == 'https://aaa.bbb.cc/x232sds' + journal_entry_object = JournalEntry.objects.get(expense_group=expense_group) + assert journal_entry_object.receipt_urls == {'txjvDntD9ZXR': 'https://aaa.bbb.cc/x232sds'} #mocking expense report creation with the file being present @@ -1693,6 +1701,8 @@ def test_upload_attachments_and_update_export(mocker, db): # asserting if the file is present lineitem = ExpenseReportLineItem.objects.get(expense_id=1) assert lineitem.netsuite_receipt_url == 'https://aaa.bbb.cc/x232sds' + expense_report_object = ExpenseReport.objects.get(expense_group=expense_group) + assert expense_report_object.receipt_urls == {'txjvDntD9ZXR': 'https://aaa.bbb.cc/x232sds'} def test_skipping_bill_creation(db, mocker):