From 4e9cd141df49e07a4bb6de924a8479cd5b54f20a Mon Sep 17 00:00:00 2001 From: Nilesh Pant <58652823+NileshPant1999@users.noreply.github.com> Date: Mon, 6 Nov 2023 12:42:45 +0530 Subject: [PATCH 01/12] add support for exportable expense group ids (#70) --- apps/fyle/helpers.py | 25 ++++++++++++++++++++++++- apps/fyle/urls.py | 6 ++++-- apps/fyle/views.py | 19 +++++++++++++++++++ tests/conftest.py | 32 +++++++++++++++++++++++++++++++- tests/test_fyle/test_views.py | 10 ++++++++++ 5 files changed, 88 insertions(+), 4 deletions(-) diff --git a/apps/fyle/helpers.py b/apps/fyle/helpers.py index 9fa49014..276f29a5 100644 --- a/apps/fyle/helpers.py +++ b/apps/fyle/helpers.py @@ -4,7 +4,8 @@ from fyle_integrations_platform_connector import PlatformConnector -from apps.workspaces.models import FyleCredential +from apps.workspaces.models import FyleCredential, ExportSetting +from apps.accounting_exports.models import AccountingExport from apps.fyle.constants import DEFAULT_FYLE_CONDITIONS @@ -128,3 +129,25 @@ def connect_to_platform(workspace_id: int) -> PlatformConnector: fyle_credentials: FyleCredential = FyleCredential.objects.get(workspace_id=workspace_id) return PlatformConnector(fyle_credentials=fyle_credentials) + + +def get_exportable_accounting_exports_ids(workspace_id: int): + """ + Get List of accounting exports ids + """ + + export_setting = ExportSetting.objects.get(workspace_id=workspace_id) + fund_source = [] + + if export_setting.reimbursable_expenses_export_type: + fund_source.append('PERSONAL') + if export_setting.credit_card_expense_export_type: + fund_source.append('CCC') + + accounting_export_ids = AccountingExport.objects.filter( + workspace_id=workspace_id, + exported_at__isnull=True, + fund_source__in=fund_source + ).values_list('id', flat=True) + + return accounting_export_ids diff --git a/apps/fyle/urls.py b/apps/fyle/urls.py index 5233f21b..6de9ff9d 100644 --- a/apps/fyle/urls.py +++ b/apps/fyle/urls.py @@ -21,7 +21,8 @@ ExpenseFilterDeleteView, CustomFieldView, FyleFieldsView, - DependentFieldSettingView + DependentFieldSettingView, + ExportableExpenseGroupsView ) @@ -31,5 +32,6 @@ path('expense_filters/', ExpenseFilterView.as_view(), name='expense-filters'), path('expense_fields/', CustomFieldView.as_view(), name='fyle-expense-fields'), path('fields/', FyleFieldsView.as_view(), name='fyle-fields'), - path('dependent_field_settings/', DependentFieldSettingView.as_view(), name='dependent-field') + path('dependent_field_settings/', DependentFieldSettingView.as_view(), name='dependent-field'), + path('exportable_accounting_exports/', ExportableExpenseGroupsView.as_view(), name='exportable-accounting-exports'), ] diff --git a/apps/fyle/views.py b/apps/fyle/views.py index 19b831ee..85868d15 100644 --- a/apps/fyle/views.py +++ b/apps/fyle/views.py @@ -1,5 +1,8 @@ import logging from rest_framework import generics +from rest_framework.views import status +from rest_framework.response import Response + from sage_desktop_api.utils import LookupFieldMixin from apps.workspaces.models import Workspace from apps.fyle.serializers import ( @@ -10,6 +13,8 @@ DependentFieldSettingSerializer ) from apps.fyle.models import ExpenseFilter, DependentFieldSetting +from apps.fyle.helpers import get_exportable_accounting_exports_ids + logger = logging.getLogger(__name__) logger.level = logging.INFO @@ -69,3 +74,17 @@ class DependentFieldSettingView(generics.CreateAPIView, generics.RetrieveUpdateA serializer_class = DependentFieldSettingSerializer lookup_field = 'workspace_id' queryset = DependentFieldSetting.objects.all() + + +class ExportableExpenseGroupsView(generics.RetrieveAPIView): + """ + List Exportable Expense Groups + """ + def get(self, request, *args, **kwargs): + + exportable_ids = get_exportable_accounting_exports_ids(workspace_id=kwargs['workspace_id']) + + return Response( + data={'exportable_expense_group_ids': exportable_ids}, + status=status.HTTP_200_OK + ) diff --git a/tests/conftest.py b/tests/conftest.py index 82941772..854bf6e7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,7 +15,8 @@ from apps.workspaces.models import ( Workspace, FyleCredential, - Sage300Credential + Sage300Credential, + ExportSetting ) from apps.fyle.models import ExpenseFilter from apps.accounting_exports.models import AccountingExport, Error, AccountingExportSummary @@ -306,3 +307,32 @@ def add_project_mappings(): detail='Sage 300 Project - Platform APIs, Id - 10081', active=True ) + + +@pytest.fixture() +@pytest.mark.django_db(databases=['default']) +def add_export_settings(): + """ + Pytest fixtue to add export_settings to a workspace + """ + + workspace_ids = [ + 1, 2, 3 + ] + + for workspace_id in workspace_ids: + ExportSetting.objects.create( + workspace_id=workspace_id, + reimbursable_expenses_export_type='BILL' if workspace_id in [1, 2] else 'JOURNAL_ENTRY', + default_bank_account_name='Accounts Payable', + default_back_account_id='1', + reimbursable_expense_state='PAYMENT_PROCESSING', + reimbursable_expense_date='current_date' if workspace_id == 1 else 'last_spent_at', + reimbursable_expense_grouped_by='REPORT' if workspace_id == 1 else 'EXPENSE', + credit_card_expense_export_type='CREDIT_CARD_PURCHASE' if workspace_id in [1, 2] else 'JOURNAL_ENTRY', + credit_card_expense_state='PAYMENT_PROCESSING', + default_ccc_credit_card_account_name='Visa', + default_ccc_credit_card_account_id='12', + credit_card_expense_grouped_by='EXPENSE' if workspace_id == 3 else 'REPORT', + credit_card_expense_date='spent_at' + ) diff --git a/tests/test_fyle/test_views.py b/tests/test_fyle/test_views.py index 6203cae0..b783b77f 100644 --- a/tests/test_fyle/test_views.py +++ b/tests/test_fyle/test_views.py @@ -100,3 +100,13 @@ def test_fyle_fields(api_client, test_connection, create_temp_workspace, add_fyl response = json.loads(response.content) assert response['results'] == data['fyle_fields_response'] + + +def test_exportable_expense_group_view(api_client, test_connection, create_temp_workspace, add_export_settings): + + access_token = test_connection.access_token + url = reverse('exportable-accounting-exports', kwargs={'workspace_id': 1}) + api_client.credentials(HTTP_AUTHORIZATION='Bearer {}'.format(access_token)) + + response = api_client.get(url) + assert response.status_code == 200 From 0a46a897febce788ad69f8d0e573c3e8a2908700 Mon Sep 17 00:00:00 2001 From: Nilesh Pant <58652823+NileshPant1999@users.noreply.github.com> Date: Mon, 6 Nov 2023 12:43:48 +0530 Subject: [PATCH 02/12] migrations fix (#65) --- apps/mappings/migrations/0001_initial.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/mappings/migrations/0001_initial.py b/apps/mappings/migrations/0001_initial.py index dd2c752a..05360017 100644 --- a/apps/mappings/migrations/0001_initial.py +++ b/apps/mappings/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.1.2 on 2023-10-27 09:30 +# Generated by Django 4.1.2 on 2023-11-02 09:56 from django.db import migrations, models import django.db.models.deletion @@ -31,7 +31,7 @@ class Migration(migrations.Migration): ('categories_version', sage_desktop_api.models.fields.StringNullField(help_text='latest sync version of categories', max_length=255, null=True)), ('cost_code_version', sage_desktop_api.models.fields.StringNullField(help_text='latest sync version of cost code', max_length=255, null=True)), ('vendor_version', sage_desktop_api.models.fields.StringNullField(help_text='latest sync version of vendor', max_length=255, null=True)), - ('workspace', models.OneToOneField(help_text='Reference to Workspace model', on_delete=django.db.models.deletion.PROTECT, to='workspaces.workspace')), + ('workspace', models.ForeignKey(help_text='Reference to Workspace model', on_delete=django.db.models.deletion.PROTECT, to='workspaces.workspace')), ], options={ 'db_table': 'import_logs', From 456dce7a031fac0958b0528b09f6d01cd087baff Mon Sep 17 00:00:00 2001 From: Nilesh Pant <58652823+NileshPant1999@users.noreply.github.com> Date: Mon, 6 Nov 2023 12:44:07 +0530 Subject: [PATCH 03/12] rename sage300 categories to cost categories (#64) --- .../migrations/0002_dependentfieldsetting.py | 8 ++--- apps/fyle/models.py | 6 ++-- apps/fyle/signals.py | 4 +-- apps/sage300/dependent_fields.py | 10 +++---- ...00categories.py => 0002_costcategories.py} | 8 ++--- apps/sage300/models.py | 30 +++++++++---------- apps/sage300/utils.py | 4 +-- 7 files changed, 35 insertions(+), 35 deletions(-) rename apps/sage300/migrations/{0002_sage300categories.py => 0002_costcategories.py} (86%) diff --git a/apps/fyle/migrations/0002_dependentfieldsetting.py b/apps/fyle/migrations/0002_dependentfieldsetting.py index 5682422c..d7b010c9 100644 --- a/apps/fyle/migrations/0002_dependentfieldsetting.py +++ b/apps/fyle/migrations/0002_dependentfieldsetting.py @@ -1,4 +1,4 @@ -# Generated by Django 4.1.2 on 2023-10-30 18:07 +# Generated by Django 4.1.2 on 2023-11-02 09:29 from django.db import migrations, models import django.db.models.deletion @@ -24,9 +24,9 @@ class Migration(migrations.Migration): ('cost_code_field_name', sage_desktop_api.models.fields.StringNotNullField(help_text='Fyle Cost Code Field Name', max_length=255)), ('cost_code_field_id', sage_desktop_api.models.fields.StringNotNullField(help_text='Fyle Cost Code Field ID', max_length=255)), ('cost_code_placeholder', models.TextField(blank=True, help_text='Placeholder for Cost code', null=True)), - ('category_field_name', sage_desktop_api.models.fields.StringNotNullField(help_text='Fyle Cost Type Field Name', max_length=255)), - ('category_field_id', sage_desktop_api.models.fields.StringNotNullField(help_text='Fyle Cost Type Field ID', max_length=255)), - ('category_placeholder', models.TextField(blank=True, help_text='Placeholder for Cost Type', null=True)), + ('cost_category_field_name', sage_desktop_api.models.fields.StringNotNullField(help_text='Fyle Cost Category Field Name', max_length=255)), + ('cost_category_field_id', sage_desktop_api.models.fields.StringNotNullField(help_text='Fyle Cost Category Field ID', max_length=255)), + ('cost_category_placeholder', models.TextField(blank=True, help_text='Placeholder for Cost Category', null=True)), ('last_successful_import_at', sage_desktop_api.models.fields.CustomDateTimeField(help_text='Last Successful Import At', null=True)), ('workspace', models.OneToOneField(help_text='Reference to Workspace model', on_delete=django.db.models.deletion.PROTECT, to='workspaces.workspace')), ], diff --git a/apps/fyle/models.py b/apps/fyle/models.py index 7dd4795a..ee7a18e0 100644 --- a/apps/fyle/models.py +++ b/apps/fyle/models.py @@ -116,9 +116,9 @@ class DependentFieldSetting(BaseModel): cost_code_field_name = StringNotNullField(help_text='Fyle Cost Code Field Name') cost_code_field_id = StringNotNullField(help_text='Fyle Cost Code Field ID') cost_code_placeholder = models.TextField(blank=True, null=True, help_text='Placeholder for Cost code') - category_field_name = StringNotNullField(max_length=255, help_text='Fyle Cost Type Field Name') - category_field_id = StringNotNullField(help_text='Fyle Cost Type Field ID') - category_placeholder = models.TextField(blank=True, null=True, help_text='Placeholder for Cost Type') + cost_category_field_name = StringNotNullField(max_length=255, help_text='Fyle Cost Category Field Name') + cost_category_field_id = StringNotNullField(help_text='Fyle Cost Category Field ID') + cost_category_placeholder = models.TextField(blank=True, null=True, help_text='Placeholder for Cost Category') last_successful_import_at = CustomDateTimeField(null=True, help_text='Last Successful Import At') class Meta: diff --git a/apps/fyle/signals.py b/apps/fyle/signals.py index 87ea6da4..26c92f1f 100644 --- a/apps/fyle/signals.py +++ b/apps/fyle/signals.py @@ -41,14 +41,14 @@ def run_pre_save_dependent_field_settings_triggers(sender, instance: DependentFi instance.cost_code_field_id = cost_code['data']['id'] - sage300_category = create_dependent_custom_field_in_fyle( + cost_category = create_dependent_custom_field_in_fyle( workspace_id=instance.workspace_id, fyle_attribute_type=instance.category_field_name, platform=platform, source_placeholder=instance.category_placeholder, parent_field_id=instance.cost_code_field_id, ) - instance.category_field_id = sage300_category['data']['id'] + instance.category_field_id = cost_category['data']['id'] @receiver(post_save, sender=DependentFieldSetting) diff --git a/apps/sage300/dependent_fields.py b/apps/sage300/dependent_fields.py index c56cf004..bd0080d2 100644 --- a/apps/sage300/dependent_fields.py +++ b/apps/sage300/dependent_fields.py @@ -8,7 +8,7 @@ from fyle_integrations_platform_connector import PlatformConnector from apps.fyle.models import DependentFieldSetting -from apps.sage300.models import Sage300Categories +from apps.sage300.models import CostCategory from apps.fyle.helpers import connect_to_platform from apps.mappings.tasks import sync_sage300_attributes @@ -70,7 +70,7 @@ def create_dependent_custom_field_in_fyle(workspace_id: int, fyle_attribute_type def post_dependent_cost_code(dependent_field_setting: DependentFieldSetting, platform: PlatformConnector, filters: Dict) -> List[str]: - projects = Sage300Categories.objects.filter(**filters).values('job_name').annotate(cost_codes=ArrayAgg('cost_code_name', distinct=True)) + projects = CostCategory.objects.filter(**filters).values('job_name').annotate(cost_codes=ArrayAgg('cost_code_name', distinct=True)) projects_from_categories = [project['job_name'] for project in projects] posted_cost_codes = [] @@ -100,9 +100,9 @@ def post_dependent_cost_code(dependent_field_setting: DependentFieldSetting, pla def post_dependent_cost_type(dependent_field_setting: DependentFieldSetting, platform: PlatformConnector, filters: Dict): - sage300_categories = Sage300Categories.objects.filter(**filters).values('cost_code_name').annotate(sage300_categories=ArrayAgg('name', distinct=True)) + cost_categories = CostCategory.objects.filter(**filters).values('cost_code_name').annotate(cost_categories=ArrayAgg('name', distinct=True)) - for category in sage300_categories: + for category in cost_categories: payload = [ { 'parent_expense_field_id': dependent_field_setting.cost_code_field_id, @@ -110,7 +110,7 @@ def post_dependent_cost_type(dependent_field_setting: DependentFieldSetting, pla 'expense_field_id': dependent_field_setting.cost_type_field_id, 'expense_field_value': cost_type, 'is_enabled': True - } for cost_type in category['sage300_categories'] + } for cost_type in category['cost_categories'] ] if payload: diff --git a/apps/sage300/migrations/0002_sage300categories.py b/apps/sage300/migrations/0002_costcategories.py similarity index 86% rename from apps/sage300/migrations/0002_sage300categories.py rename to apps/sage300/migrations/0002_costcategories.py index 4029401a..05c5ae44 100644 --- a/apps/sage300/migrations/0002_sage300categories.py +++ b/apps/sage300/migrations/0002_costcategories.py @@ -1,4 +1,4 @@ -# Generated by Django 4.1.2 on 2023-10-31 06:48 +# Generated by Django 4.1.2 on 2023-11-02 09:29 from django.db import migrations, models import django.db.models.deletion @@ -14,7 +14,7 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name='Sage300Categories', + name='CostCategory', fields=[ ('created_at', models.DateTimeField(auto_now_add=True, help_text='Created at datetime')), ('updated_at', models.DateTimeField(auto_now=True, help_text='Updated at datetime')), @@ -24,12 +24,12 @@ class Migration(migrations.Migration): ('cost_code_id', sage_desktop_api.models.fields.StringNotNullField(help_text='Sage300 Cost Code Id', max_length=255)), ('cost_code_name', sage_desktop_api.models.fields.StringNotNullField(help_text='Sage300 Cost Code Name', max_length=255)), ('name', sage_desktop_api.models.fields.StringNotNullField(help_text='Sage300 Cost Type Name', max_length=255)), - ('category_id', sage_desktop_api.models.fields.StringNotNullField(help_text='Sage300 Category Id', max_length=255)), + ('cost_category_id', sage_desktop_api.models.fields.StringNotNullField(help_text='Sage300 Category Id', max_length=255)), ('status', sage_desktop_api.models.fields.BooleanFalseField(default=True, help_text='Sage300 Cost Type Status')), ('workspace', models.ForeignKey(help_text='Reference to Workspace model', on_delete=django.db.models.deletion.PROTECT, to='workspaces.workspace')), ], options={ - 'db_table': 'sage300_categories', + 'db_table': 'cost_category', }, ), ] diff --git a/apps/sage300/models.py b/apps/sage300/models.py index 01c9e956..58522e71 100644 --- a/apps/sage300/models.py +++ b/apps/sage300/models.py @@ -102,9 +102,9 @@ class Meta: db_table = 'direct_cost' -class Sage300Categories(BaseForeignWorkspaceModel): +class CostCategory(BaseForeignWorkspaceModel): """ - Categories Table Model Class + Cost Categories Table Model Class """ id = models.AutoField(primary_key=True) @@ -113,11 +113,11 @@ class Sage300Categories(BaseForeignWorkspaceModel): cost_code_id = StringNotNullField(help_text='Sage300 Cost Code Id') cost_code_name = StringNotNullField(help_text='Sage300 Cost Code Name') name = StringNotNullField(help_text='Sage300 Cost Type Name') - category_id = StringNotNullField(help_text='Sage300 Category Id') + cost_category_id = StringNotNullField(help_text='Sage300 Category Id') status = BooleanFalseField(help_text='Sage300 Cost Type Status') class Meta: - db_table = 'sage300_categories' + db_table = 'cost_category' @staticmethod def bulk_create_or_update(categories_generator: List[Dict], workspace_id: int): @@ -136,7 +136,7 @@ def bulk_create_or_update(categories_generator: List[Dict], workspace_id: int): 'workspace_id': workspace_id } - existing_categories = Sage300Categories.objects.filter(**filters).values( + existing_categories = CostCategory.objects.filter(**filters).values( 'id', 'category_id', 'name', @@ -154,8 +154,8 @@ def bulk_create_or_update(categories_generator: List[Dict], workspace_id: int): 'status': existing_category['status'], } - category_to_be_created = [] - category_to_be_updated = [] + cost_category_to_be_created = [] + cost_category_to_be_updated = [] # Retrieve job names and cost code names in a single query cost_code_ids = [category.cost_code_id for category in list_of_categories] @@ -167,7 +167,7 @@ def bulk_create_or_update(categories_generator: List[Dict], workspace_id: int): for category in list_of_categories: job_name = job_name_mapping.get(category.job_id) cost_code_name = cost_code_name_mapping.get(category.cost_code_id) - category_object = Sage300Categories( + category_object = CostCategory( job_id=category.job_id, job_name=job_name, cost_code_id=category.cost_code_id, @@ -179,20 +179,20 @@ def bulk_create_or_update(categories_generator: List[Dict], workspace_id: int): ) if category.id not in existing_cost_type_record_numbers: - category_to_be_created.append(category_object) + cost_category_to_be_created.append(category_object) elif category.id in primary_key_map.keys() and ( category.name != primary_key_map[category.id]['name'] or category.is_active != primary_key_map[category.id]['status'] ): category_object.id = primary_key_map[category.id]['category_id'] - category_to_be_updated.append(category_object) + cost_category_to_be_updated.append(category_object) - if category_to_be_created: - Sage300Categories.objects.bulk_create(category_to_be_created, batch_size=2000) + if cost_category_to_be_created: + CostCategory.objects.bulk_create(cost_category_to_be_created, batch_size=2000) - if category_to_be_updated: - Sage300Categories.objects.bulk_update( - category_to_be_updated, fields=[ + if cost_category_to_be_updated: + CostCategory.objects.bulk_update( + cost_category_to_be_updated, fields=[ 'job_id', 'job_name', 'cost_code_id', 'cost_code_name', 'name', 'status', 'category_id' ], diff --git a/apps/sage300/utils.py b/apps/sage300/utils.py index 91c95c3b..d4cf1b1a 100644 --- a/apps/sage300/utils.py +++ b/apps/sage300/utils.py @@ -1,7 +1,7 @@ from fyle_accounting_mappings.models import DestinationAttribute from apps.workspaces.models import Sage300Credential from sage_desktop_sdk.sage_desktop_sdk import SageDesktopSDK -from apps.sage300.models import Sage300Categories +from apps.sage300.models import CostCategory class SageDesktopConnector: @@ -147,4 +147,4 @@ def sync_categories(self): Synchronize categories from Sage Desktop SDK to your application """ categories = self.connection.categories.get_all_categories() - Sage300Categories.bulk_create_or_update(categories, self.workspace_id) + CostCategory.bulk_create_or_update(categories, self.workspace_id) From cd5216c695f2b8d63256c686134225af98417392 Mon Sep 17 00:00:00 2001 From: Nilesh Pant <58652823+NileshPant1999@users.noreply.github.com> Date: Mon, 6 Nov 2023 12:44:41 +0530 Subject: [PATCH 04/12] add support for scheduling tasks (#69) --- apps/fyle/signals.py | 7 +- apps/fyle/views.py | 2 - apps/mappings/apps.py | 4 + apps/mappings/imports/modules/base.py | 10 ++- apps/mappings/imports/queues.py | 43 ++++++++++ apps/mappings/imports/schedules.py | 83 +++++++++++++++++++ apps/mappings/imports/tasks.py | 25 ++++++ apps/mappings/signals.py | 115 ++++++++++++++++++++++++++ apps/mappings/urls.py | 20 +++++ apps/sage300/serializers.py | 2 +- apps/workspaces/urls.py | 1 + 11 files changed, 304 insertions(+), 8 deletions(-) create mode 100644 apps/mappings/signals.py diff --git a/apps/fyle/signals.py b/apps/fyle/signals.py index 26c92f1f..cfdfb45a 100644 --- a/apps/fyle/signals.py +++ b/apps/fyle/signals.py @@ -7,9 +7,11 @@ from django.dispatch import receiver from fyle_integrations_platform_connector import PlatformConnector -from apps.workspaces.models import FyleCredential +from apps.workspaces.models import FyleCredential, ImportSetting from apps.fyle.models import DependentFieldSetting from apps.sage300.dependent_fields import create_dependent_custom_field_in_fyle +from apps.mappings.imports.schedules import schedule_or_delete_fyle_import_tasks + logger = logging.getLogger(__name__) logger.level = logging.INFO @@ -58,4 +60,5 @@ def run_post_save_dependent_field_settings_triggers(sender, instance: DependentF :param instance: Row instance of Sender Class :return: None """ - pass + import_settings = ImportSetting.objects.filter(workspace_id=instance.workspace_id).first() + schedule_or_delete_fyle_import_tasks(import_settings) diff --git a/apps/fyle/views.py b/apps/fyle/views.py index 85868d15..5153f414 100644 --- a/apps/fyle/views.py +++ b/apps/fyle/views.py @@ -69,8 +69,6 @@ class DependentFieldSettingView(generics.CreateAPIView, generics.RetrieveUpdateA """ Dependent Field view """ - authentication_classes = [] - permission_classes = [] serializer_class = DependentFieldSettingSerializer lookup_field = 'workspace_id' queryset = DependentFieldSetting.objects.all() diff --git a/apps/mappings/apps.py b/apps/mappings/apps.py index 7ef2ccf4..43c290b7 100644 --- a/apps/mappings/apps.py +++ b/apps/mappings/apps.py @@ -4,3 +4,7 @@ class MappingsConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "apps.mappings" + + def ready(self): + super(MappingsConfig, self).ready() + import apps.mappings.signals # noqa diff --git a/apps/mappings/imports/modules/base.py b/apps/mappings/imports/modules/base.py index 27bf6024..f503ff41 100644 --- a/apps/mappings/imports/modules/base.py +++ b/apps/mappings/imports/modules/base.py @@ -191,6 +191,8 @@ def construct_payload_and_import_to_fyle( destination_attributes_count = DestinationAttribute.objects.filter(**filters).count() + is_auto_sync_status_allowed = self.get_auto_sync_permission() + # If there are no destination attributes, mark the import as complete if destination_attributes_count == 0: import_log.status = 'COMPLETE' @@ -209,7 +211,8 @@ def construct_payload_and_import_to_fyle( for paginated_destination_attributes, is_last_batch in destination_attributes_generator: fyle_payload = self.setup_fyle_payload_creation( - paginated_destination_attributes=paginated_destination_attributes + paginated_destination_attributes=paginated_destination_attributes, + is_auto_sync_status_allowed=is_auto_sync_status_allowed ) self.post_to_fyle_and_sync( @@ -236,7 +239,8 @@ def get_destination_attributes_generator(self, destination_attributes_count: int def setup_fyle_payload_creation( self, - paginated_destination_attributes: List[DestinationAttribute] + paginated_destination_attributes: List[DestinationAttribute], + is_auto_sync_status_allowed: bool ): """ Setup Fyle Payload Creation @@ -247,7 +251,7 @@ def setup_fyle_payload_creation( paginated_destination_attribute_values = [attribute.value for attribute in paginated_destination_attributes] existing_expense_attributes_map = self.get_existing_fyle_attributes(paginated_destination_attribute_values) - return self.construct_fyle_payload(paginated_destination_attributes, existing_expense_attributes_map) + return self.construct_fyle_payload(paginated_destination_attributes, existing_expense_attributes_map, is_auto_sync_status_allowed) def get_existing_fyle_attributes(self, paginated_destination_attribute_values: List[str]): """ diff --git a/apps/mappings/imports/queues.py b/apps/mappings/imports/queues.py index e69de29b..fea903a2 100644 --- a/apps/mappings/imports/queues.py +++ b/apps/mappings/imports/queues.py @@ -0,0 +1,43 @@ +from django_q.tasks import Chain +from fyle_accounting_mappings.models import MappingSetting +from apps.workspaces.models import ImportSetting + + +def chain_import_fields_to_fyle(workspace_id): + """ + Chain import fields to Fyle + :param workspace_id: Workspace Id + """ + mapping_settings = MappingSetting.objects.filter(workspace_id=workspace_id, import_to_fyle=True) + custom_field_mapping_settings = MappingSetting.objects.filter(workspace_id=workspace_id, is_custom=True, import_to_fyle=True) + import_settings = ImportSetting.objects.get(workspace_id=workspace_id) + chain = Chain() + + if import_settings.import_categories: + chain.append( + 'apps.mappings.imports.tasks.trigger_import_via_schedule', + workspace_id, + 'ACCOUNT', + 'CATEGORY' + ) + + for mapping_setting in mapping_settings: + if mapping_setting.source_field in ['PROJECT', 'COST_CENTER']: + chain.append( + 'apps.mappings.imports.tasks.trigger_import_via_schedule', + workspace_id, + mapping_setting.destination_field, + mapping_setting.source_field + ) + + for custom_fields_mapping_setting in custom_field_mapping_settings: + chain.append( + 'apps.mappings.imports.tasks.trigger_import_via_schedule', + workspace_id, + custom_fields_mapping_setting.destination_field, + custom_fields_mapping_setting.source_field, + True + ) + + if chain.length() > 0: + chain.run() diff --git a/apps/mappings/imports/schedules.py b/apps/mappings/imports/schedules.py index e69de29b..ea67ca0e 100644 --- a/apps/mappings/imports/schedules.py +++ b/apps/mappings/imports/schedules.py @@ -0,0 +1,83 @@ +from datetime import datetime +from django_q.models import Schedule +from fyle_accounting_mappings.models import MappingSetting + +from apps.fyle.models import DependentFieldSetting +from apps.workspaces.models import ImportSetting + + +def schedule_or_delete_dependent_field_tasks(import_settings: ImportSetting): + """ + :param configuration: Workspace Configuration Instance + :return: None + """ + project_mapping = MappingSetting.objects.filter( + source_field='PROJECT', + workspace_id=import_settings.workspace_id, + import_to_fyle=True + ).first() + dependent_fields = DependentFieldSetting.objects.filter(workspace_id=import_settings.workspace_id, is_import_enabled=True).first() + + if project_mapping and dependent_fields: + start_datetime = datetime.now() + Schedule.objects.update_or_create( + func='apps.mappings.tasks.auto_import_and_map_fyle_fields', + args='{}'.format(import_settings.workspace_id), + defaults={ + 'schedule_type': Schedule.MINUTES, + 'minutes': 24 * 60, + 'next_run': start_datetime + } + ) + elif not (project_mapping and dependent_fields): + Schedule.objects.filter( + func='apps.mappings.tasks.auto_import_and_map_fyle_fields', + args='{}'.format(import_settings.workspace_id) + ).delete() + + +def schedule_or_delete_fyle_import_tasks(import_settings: ImportSetting, mapping_setting_instance: MappingSetting = None): + """ + Schedule or delete Fyle import tasks based on the import settingss. + :param import_settingss: Workspace ImportSetting Instance + :param instance: Mapping Setting Instance + :return: None + """ + task_to_be_scheduled = None + # Check if there is a task to be scheduled + if mapping_setting_instance and mapping_setting_instance.import_to_fyle: + task_to_be_scheduled = mapping_setting_instance + + if task_to_be_scheduled or import_settings.import_categories: + Schedule.objects.update_or_create( + func='apps.mappings.imports.queues.chain_import_fields_to_fyle', + args='{}'.format(import_settings.workspace_id), + defaults={ + 'schedule_type': Schedule.MINUTES, + 'minutes': 24 * 60, + 'next_run': datetime.now() + } + ) + return + + import_fields_count = MappingSetting.objects.filter( + import_to_fyle=True, + workspace_id=import_settings.workspace_id, + source_field__in=['CATEGORY', 'PROJECT', 'COST_CENTER'] + ).count() + + custom_field_import_fields_count = MappingSetting.objects.filter( + import_to_fyle=True, + workspace_id=import_settings.workspace_id, + is_custom=True + ).count() + + # If the import fields count is 0, delete the schedule + if import_fields_count == 0 and custom_field_import_fields_count == 0: + Schedule.objects.filter( + func='apps.mappings.imports.queues.chain_import_fields_to_fyle', + args='{}'.format(import_settings.workspace_id) + ).delete() + + # Schedule or delete dependent field tasks + schedule_or_delete_dependent_field_tasks(import_settings=import_settings) diff --git a/apps/mappings/imports/tasks.py b/apps/mappings/imports/tasks.py index 34c7b7c0..b949b17f 100644 --- a/apps/mappings/imports/tasks.py +++ b/apps/mappings/imports/tasks.py @@ -1,7 +1,12 @@ +from django_q.tasks import Chain + +from fyle_accounting_mappings.models import MappingSetting + from apps.mappings.models import ImportLog from apps.mappings.imports.modules.categories import Category from apps.mappings.imports.modules.projects import Project from apps.mappings.imports.modules.expense_custom_fields import ExpenseCustomField +from apps.fyle.models import DependentFieldSetting SOURCE_FIELD_CLASS_MAP = { @@ -27,3 +32,23 @@ def trigger_import_via_schedule(workspace_id: int, destination_field: str, sourc module_class = SOURCE_FIELD_CLASS_MAP[source_field] item = module_class(workspace_id, destination_field, sync_after) item.trigger_import() + + +def auto_import_and_map_fyle_fields(workspace_id): + """ + Auto import and map fyle fields + """ + project_mapping = MappingSetting.objects.filter( + source_field='PROJECT', + workspace_id=workspace_id, + import_to_fyle=True + ).first() + dependent_fields = DependentFieldSetting.objects.filter(workspace_id=workspace_id, is_import_enabled=True).first() + + chain = Chain() + + if project_mapping and dependent_fields: + chain.append('apps.sage_intacct.dependent_fields.import_dependent_fields_to_fyle', workspace_id) + + if chain.length() > 0: + chain.run() diff --git a/apps/mappings/signals.py b/apps/mappings/signals.py new file mode 100644 index 00000000..8366f2e5 --- /dev/null +++ b/apps/mappings/signals.py @@ -0,0 +1,115 @@ +import logging +from datetime import timedelta, datetime, timezone +from django.db.models.signals import post_save, pre_save +from django.dispatch import receiver + +from rest_framework.exceptions import ValidationError +from fyle_accounting_mappings.models import MappingSetting +from fyle_integrations_platform_connector import PlatformConnector +from fyle.platform.exceptions import WrongParamsError + +from apps.mappings.imports.schedules import schedule_or_delete_fyle_import_tasks +from apps.workspaces.models import ImportSetting +from apps.mappings.models import ImportLog +from apps.workspaces.models import FyleCredential +from apps.mappings.imports.modules.expense_custom_fields import ExpenseCustomField + + +logger = logging.getLogger(__name__) + + +@receiver(post_save, sender=MappingSetting) +def run_post_mapping_settings_triggers(sender, instance: MappingSetting, **kwargs): + """ + :param sender: Sender Class + :param instance: Row instance of Sender Class + :return: None + """ + import_settings = ImportSetting.objects.filter(workspace_id=instance.workspace_id).first() + + if instance.source_field == 'PROJECT': + schedule_or_delete_fyle_import_tasks(import_settings, instance) + + if instance.source_field == 'COST_CENTER': + schedule_or_delete_fyle_import_tasks(import_settings, instance) + + if instance.is_custom: + schedule_or_delete_fyle_import_tasks(import_settings, instance) + + +@receiver(pre_save, sender=MappingSetting) +def run_pre_mapping_settings_triggers(sender, instance: MappingSetting, **kwargs): + """ + :param sender: Sender Class + :param instance: Row instance of Sender Class + :return: None + """ + default_attributes = ['EMPLOYEE', 'CATEGORY', 'PROJECT', 'COST_CENTER'] + + instance.source_field = instance.source_field.upper().replace(' ', '_') + + if instance.source_field not in default_attributes and instance.import_to_fyle: + # TODO: sync intacct fields before we upload custom field + try: + workspace_id = int(instance.workspace_id) + # Checking is import_log exists or not if not create one + import_log, is_created = ImportLog.objects.get_or_create( + workspace_id=workspace_id, + attribute_type=instance.source_field, + defaults={ + 'status': 'IN_PROGRESS' + } + ) + + last_successful_run_at = None + if import_log and not is_created: + last_successful_run_at = import_log.last_successful_run_at if import_log.last_successful_run_at else None + time_difference = datetime.now() - timedelta(minutes=32) + offset_aware_time_difference = time_difference.replace(tzinfo=timezone.utc) + + # if the import_log is present and the last_successful_run_at is less than 30mins then we need to update it + # so that the schedule can run + if last_successful_run_at and offset_aware_time_difference\ + and (offset_aware_time_difference < last_successful_run_at): + import_log.last_successful_run_at = offset_aware_time_difference + last_successful_run_at = offset_aware_time_difference + import_log.save() + + # Creating the expense_custom_field object with the correct last_successful_run_at value + expense_custom_field = ExpenseCustomField( + workspace_id=workspace_id, + source_field=instance.source_field, + destination_field=instance.destination_field, + sync_after=last_successful_run_at + ) + + fyle_credentials = FyleCredential.objects.get(workspace_id=workspace_id) + platform = PlatformConnector(fyle_credentials=fyle_credentials) + + # setting the import_log status to IN_PROGRESS + import_log.status = 'IN_PROGRESS' + import_log.save() + + expense_custom_field.construct_payload_and_import_to_fyle(platform, import_log) + expense_custom_field.sync_expense_attributes(platform) + + # NOTE: We are not setting the import_log status to COMPLETE + # since the post_save trigger will run the import again in async manner + + except WrongParamsError as error: + logger.error( + 'Error while creating %s workspace_id - %s in Fyle %s %s', + instance.source_field, instance.workspace_id, error.message, {'error': error.response} + ) + if error.response and 'message' in error.response: + raise ValidationError({ + 'message': error.response['message'], + 'field_name': instance.source_field + }) + + # setting the import_log.last_successful_run_at to -30mins for the post_save_trigger + import_log = ImportLog.objects.filter(workspace_id=workspace_id, attribute_type=instance.source_field).first() + if import_log.last_successful_run_at: + last_successful_run_at = import_log.last_successful_run_at - timedelta(minutes=30) + import_log.last_successful_run_at = last_successful_run_at + import_log.save() diff --git a/apps/mappings/urls.py b/apps/mappings/urls.py index e69de29b..1c6e39ab 100644 --- a/apps/mappings/urls.py +++ b/apps/mappings/urls.py @@ -0,0 +1,20 @@ +"""fyle_intacct_api URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/3.0/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.urls import path, include + +urlpatterns = [ + path('', include('fyle_accounting_mappings.urls')) +] diff --git a/apps/sage300/serializers.py b/apps/sage300/serializers.py index a8da3b31..6a29aadf 100644 --- a/apps/sage300/serializers.py +++ b/apps/sage300/serializers.py @@ -10,6 +10,7 @@ from apps.workspaces.models import Workspace, Sage300Credential from apps.sage300.helpers import sync_dimensions, check_interval_and_sync_dimension + logger = logging.getLogger(__name__) logger.level = logging.INFO @@ -78,7 +79,6 @@ class Sage300FieldSerializer(serializers.Serializer): display_name = serializers.CharField() def format_sage300_fields(self, workspace_id): - attribute_types = [ "VENDOR", "ACCOUNT", diff --git a/apps/workspaces/urls.py b/apps/workspaces/urls.py index ea695810..67318364 100644 --- a/apps/workspaces/urls.py +++ b/apps/workspaces/urls.py @@ -39,6 +39,7 @@ path('/sage_300/', include('apps.sage300.urls')), path('/fyle/', include('apps.fyle.urls')), path('/accounting_exports/', include('apps.accounting_exports.urls')), + path('/mappings/', include('apps.mappings.urls')) ] urlpatterns = [] From 015355b23acca59bdd31070923330ee26056b638 Mon Sep 17 00:00:00 2001 From: Nilesh Pant <58652823+NileshPant1999@users.noreply.github.com> Date: Mon, 6 Nov 2023 12:46:15 +0530 Subject: [PATCH 05/12] add version support in the sdk (#71) --- apps/mappings/migrations/0001_initial.py | 27 ++++++-- .../0002_alter_importlog_workspace.py | 20 ------ apps/mappings/models.py | 27 +++++--- sage_desktop_sdk/apis/accounts.py | 19 +++-- sage_desktop_sdk/apis/categories.py | 24 +++++-- sage_desktop_sdk/apis/commitments.py | 45 +++++++++--- sage_desktop_sdk/apis/cost_codes.py | 23 +++++-- sage_desktop_sdk/apis/jobs.py | 69 +++++++++++++++---- sage_desktop_sdk/apis/vendors.py | 46 ++++++++++--- 9 files changed, 225 insertions(+), 75 deletions(-) delete mode 100644 apps/mappings/migrations/0002_alter_importlog_workspace.py diff --git a/apps/mappings/migrations/0001_initial.py b/apps/mappings/migrations/0001_initial.py index 05360017..ba650f58 100644 --- a/apps/mappings/migrations/0001_initial.py +++ b/apps/mappings/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.1.2 on 2023-11-02 09:56 +# Generated by Django 4.1.2 on 2023-11-06 07:01 from django.db import migrations, models import django.db.models.deletion @@ -14,6 +14,26 @@ class Migration(migrations.Migration): ] operations = [ + migrations.CreateModel( + name='Version', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True, help_text='Created at datetime')), + ('updated_at', models.DateTimeField(auto_now=True, help_text='Updated at datetime')), + ('account', sage_desktop_api.models.fields.IntegerNullField(help_text='version for account', null=True)), + ('job', sage_desktop_api.models.fields.IntegerNullField(help_text='version for job', null=True)), + ('standard_category', sage_desktop_api.models.fields.IntegerNullField(help_text='version for standard category', null=True)), + ('standard_cost_code', sage_desktop_api.models.fields.IntegerNullField(help_text='version for standard costcode', null=True)), + ('job_category', sage_desktop_api.models.fields.IntegerNullField(help_text='version for job category', null=True)), + ('cost_code', sage_desktop_api.models.fields.IntegerNullField(help_text='version for costcode', null=True)), + ('vendor', sage_desktop_api.models.fields.IntegerNullField(help_text='version for vendor', null=True)), + ('commitment', sage_desktop_api.models.fields.IntegerNullField(help_text='version for commitment', null=True)), + ('workspace', models.OneToOneField(help_text='Reference to Workspace model', on_delete=django.db.models.deletion.PROTECT, to='workspaces.workspace')), + ], + options={ + 'abstract': False, + }, + ), migrations.CreateModel( name='ImportLog', fields=[ @@ -26,11 +46,6 @@ class Migration(migrations.Migration): ('total_batches_count', sage_desktop_api.models.fields.IntegerNotNullField(default=0, help_text='Queued batches')), ('processed_batches_count', sage_desktop_api.models.fields.IntegerNotNullField(default=0, help_text='Processed batches')), ('last_successful_run_at', sage_desktop_api.models.fields.CustomDateTimeField(help_text='Last successful run', null=True)), - ('accounts_version', sage_desktop_api.models.fields.StringNullField(help_text='latest sync version of accounts', max_length=255, null=True)), - ('job_version', sage_desktop_api.models.fields.StringNullField(help_text='latest sync version of job', max_length=255, null=True)), - ('categories_version', sage_desktop_api.models.fields.StringNullField(help_text='latest sync version of categories', max_length=255, null=True)), - ('cost_code_version', sage_desktop_api.models.fields.StringNullField(help_text='latest sync version of cost code', max_length=255, null=True)), - ('vendor_version', sage_desktop_api.models.fields.StringNullField(help_text='latest sync version of vendor', max_length=255, null=True)), ('workspace', models.ForeignKey(help_text='Reference to Workspace model', on_delete=django.db.models.deletion.PROTECT, to='workspaces.workspace')), ], options={ diff --git a/apps/mappings/migrations/0002_alter_importlog_workspace.py b/apps/mappings/migrations/0002_alter_importlog_workspace.py deleted file mode 100644 index 71b2e31b..00000000 --- a/apps/mappings/migrations/0002_alter_importlog_workspace.py +++ /dev/null @@ -1,20 +0,0 @@ -# Generated by Django 4.1.2 on 2023-11-02 09:05 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('workspaces', '0002_sage300credential_importsetting_fylecredential_and_more'), - ('mappings', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='importlog', - name='workspace', - field=models.ForeignKey(help_text='Reference to Workspace model', on_delete=django.db.models.deletion.PROTECT, to='workspaces.workspace'), - ), - ] diff --git a/apps/mappings/models.py b/apps/mappings/models.py index d26a7548..c63e38ca 100644 --- a/apps/mappings/models.py +++ b/apps/mappings/models.py @@ -1,13 +1,13 @@ from django.db import models -from apps.workspaces.models import BaseForeignWorkspaceModel +from apps.workspaces.models import BaseForeignWorkspaceModel, BaseModel from sage_desktop_api.models.fields import ( CustomJsonField, StringNotNullField, StringOptionsField, IntegerNotNullField, - StringNullField, - CustomDateTimeField + CustomDateTimeField, + IntegerNullField ) IMPORT_STATUS_CHOICES = ( @@ -30,12 +30,23 @@ class ImportLog(BaseForeignWorkspaceModel): total_batches_count = IntegerNotNullField(help_text='Queued batches', default=0) processed_batches_count = IntegerNotNullField(help_text='Processed batches', default=0) last_successful_run_at = CustomDateTimeField(help_text='Last successful run') - accounts_version = StringNullField(help_text='latest sync version of accounts') - job_version = StringNullField(help_text='latest sync version of job') - categories_version = StringNullField(help_text='latest sync version of categories') - cost_code_version = StringNullField(help_text='latest sync version of cost code') - vendor_version = StringNullField(help_text='latest sync version of vendor') class Meta: db_table = 'import_logs' unique_together = ('workspace', 'attribute_type') + + +class Version(BaseModel): + """ + Table to store versions + """ + + id = models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False) + account = IntegerNullField(help_text='version for account') + job = IntegerNullField(help_text='version for job') + standard_category = IntegerNullField(help_text='version for standard category') + standard_cost_code = IntegerNullField(help_text='version for standard costcode') + job_category = IntegerNullField(help_text='version for job category') + cost_code = IntegerNullField(help_text='version for costcode') + vendor = IntegerNullField(help_text='version for vendor') + commitment = IntegerNullField(help_text='version for commitment') diff --git a/sage_desktop_sdk/apis/accounts.py b/sage_desktop_sdk/apis/accounts.py index 488820e4..dfbff1a5 100644 --- a/sage_desktop_sdk/apis/accounts.py +++ b/sage_desktop_sdk/apis/accounts.py @@ -10,11 +10,22 @@ class Accounts(Client): GET_ACCOUNTS = '/GeneralLedger/Api/V1/Account.svc/accounts' - def get_all(self): + def get_all(self, version: int = None): """ - Get all Attachables - :return: List of Dicts in Attachable Schema + Get all accounts. + :param version: API version + :type version: int + :return: A generator yielding accounts in the Attachable Schema + :rtype: generator of Account objects """ - accounts = self._query_get_all(Accounts.GET_ACCOUNTS) + if version: + # Append the version query parameter if provided + query_params = '?version={0}'.format(version) + endpoint = Accounts.GET_ACCOUNTS + query_params + else: + endpoint = Accounts.GET_ACCOUNTS + + # Query the API to get all accounts + accounts = self._query_get_all(endpoint) for account in accounts: yield Account.from_dict(account) diff --git a/sage_desktop_sdk/apis/categories.py b/sage_desktop_sdk/apis/categories.py index 7b65ad18..4c9ed95f 100644 --- a/sage_desktop_sdk/apis/categories.py +++ b/sage_desktop_sdk/apis/categories.py @@ -10,12 +10,26 @@ class Categories(Client): GET_CATEGORIES = '/JobCosting/Api/V1/JobCost.svc/jobs/categories' - def get_all_categories(self): + def get_all_categories(self, version: int = None): """ - Get all Jobs - :return: List of Dicts in Jobs Schema + Get all job categories. + + :param version: API version + :type version: int + + :return: A generator yielding job categories in the Jobs Schema + :rtype: generator of Category objects """ - categories = self._query_get_all(Categories.GET_CATEGORIES) + if version: + # Append the version query parameter if provided + query_params = '?version={0}'.format(version) + endpoint = Categories.GET_CATEGORIES + query_params + else: + endpoint = Categories.GET_CATEGORIES + + # Query the API to get all job categories + categories = self._query_get_all(endpoint) + for category in categories: - print('catefor', category) + # Convert each category dictionary to a Category object and yield it yield Category.from_dict(category) diff --git a/sage_desktop_sdk/apis/commitments.py b/sage_desktop_sdk/apis/commitments.py index 79222524..77a25cee 100644 --- a/sage_desktop_sdk/apis/commitments.py +++ b/sage_desktop_sdk/apis/commitments.py @@ -12,18 +12,47 @@ class Commitments(Client): GET_COMMITMENT_ITEMS = '/JobCosting/Api/V1/Commitment.svc/commitments/items/synchronize?commitment={}' GET_COMMITMENTS = '/JobCosting/Api/V1/Commitment.svc/commitments' - def get_all(self): + def get_all(self, version: int = None): """ - Get all Vendors - :return: List of Dicts in Vendors Schema + Get all commitments. + + :param version: API version + :type version: int + + :return: A generator yielding commitments in the Commitments Schema + :rtype: generator of Commitment objects """ - commitments = self._query_get_all(Commitments.GET_COMMITMENTS) + if version: + # Append the version query parameter if provided + query_params = '?version={0}'.format(version) + endpoint = Commitments.GET_COMMITMENTS + query_params + else: + endpoint = Commitments.GET_COMMITMENTS + + # Query the API to get all commitments + commitments = self._query_get_all(endpoint) + for commitment in commitments: + # Convert each commitment dictionary to a Commitment object and yield it yield Commitment.from_dict(commitment) - def get_commitment_items(self, commitment_id: str): + def get_commitment_items(self, commitment_id: str, version: int = None): """ - Get Commitment By Id - :return: Dicts in Commitment Schema + Get commitment items by ID. + + :param commitment_id: Commitment ID + :type commitment_id: str + :param version: API version + :type version: int + + :return: A dictionary in the Commitment Schema + :rtype: Commitment object """ - return self._query_get_by_id(Commitments.GET_COMMITMENT_ITEMS.format(commitment_id)) + if version: + # Append the version query parameter if provided + query_params = '?version={0}'.format(version) + endpoint = Commitments.GET_COMMITMENT_ITEMS.format(commitment_id) + query_params + else: + endpoint = Commitments.GET_COMMITMENT_ITEMS.format(commitment_id) + + return self._query_get_by_id(endpoint) diff --git a/sage_desktop_sdk/apis/cost_codes.py b/sage_desktop_sdk/apis/cost_codes.py index f17ba228..f2a6b9b8 100644 --- a/sage_desktop_sdk/apis/cost_codes.py +++ b/sage_desktop_sdk/apis/cost_codes.py @@ -10,11 +10,26 @@ class CostCodes(Client): GET_COST_CODE = '/JobCosting/Api/V1/JobCost.svc/jobs/costcodes' - def get_all_costcodes(self): + def get_all_costcodes(self, version: int = None): """ - Get all Cost Code - :return: List of Dicts in Cost Code Schema + Get all cost codes. + + :param version: API version + :type version: int + + :return: A generator yielding cost codes in the Cost Code Schema + :rtype: generator of CostCode objects """ - cost_codes = self._query_get_all(CostCodes.GET_COST_CODE) + if version: + # Append the version query parameter if provided + query_params = '?version={0}'.format(version) + endpoint = CostCodes.GET_COST_CODE + query_params + else: + endpoint = CostCodes.GET_COST_CODE + + # Query the API to get all cost codes + cost_codes = self._query_get_all(endpoint) + for cost_code in cost_codes: + # Convert each cost code dictionary to a CostCode object and yield it yield CostCode.from_dict(cost_code) diff --git a/sage_desktop_sdk/apis/jobs.py b/sage_desktop_sdk/apis/jobs.py index e9b8ca73..12a6360a 100644 --- a/sage_desktop_sdk/apis/jobs.py +++ b/sage_desktop_sdk/apis/jobs.py @@ -12,29 +12,74 @@ class Jobs(Client): GET_COST_CODES = '/JobCosting/Api/V1/JobCost.svc/costcodes' GET_CATEGORIES = '/JobCosting/Api/V1/JobCost.svc/categories' - def get_all_jobs(self): + def get_all_jobs(self, version: int = None): """ - Get all Jobs - :return: List of Dicts in Jobs Schema + Get all jobs. + + :param version: API version + :type version: int + + :return: A generator yielding jobs in the Jobs Schema + :rtype: generator of Job objects """ - jobs = self._query_get_all(Jobs.GET_JOBS) + if version: + # Append the version query parameter if provided + query_params = '?version={0}'.format(version) + endpoint = Jobs.GET_JOBS + query_params + else: + endpoint = Jobs.GET_JOBS + + # Query the API to get all jobs + jobs = self._query_get_all(endpoint) + for job in jobs: + # Convert each job dictionary to a Job object and yield it yield Job.from_dict(job) - def get_standard_costcodes(self): + def get_standard_costcodes(self, version: int = None): """ - Get all standard Cost Codes - :return: List of Dicts in cost code Schema + Get all standard cost codes. + + :param version: API version + :type version: int + + :return: A generator yielding standard cost codes in the Cost Code Schema + :rtype: generator of StandardCostCode objects """ - costcodes = self._query_get_all(Jobs.GET_COST_CODES) + if version: + # Append the version query parameter if provided + query_params = '?version={0}'.format(version) + endpoint = Jobs.GET_COST_CODES + query_params + else: + endpoint = Jobs.GET_COST_CODES + + # Query the API to get all standard cost codes + costcodes = self._query_get_all(endpoint) + for costcode in costcodes: + # Convert each cost code dictionary to a StandardCostCode object and yield it yield StandardCostCode.from_dict(costcode) - def get_standard_categories(self): + def get_standard_categories(self, version: int = None): """ - Get all standard Categories - :return: List of Dicts in Categories Schema + Get all standard categories. + + :param version: API version + :type version: int + + :return: A generator yielding standard categories in the Categories Schema + :rtype: generator of StandardCategory objects """ - categories = self._query_get_all(Jobs.GET_CATEGORIES) + if version: + # Append the version query parameter if provided + query_params = '?version={0}'.format(version) + endpoint = Jobs.GET_CATEGORIES + query_params + else: + endpoint = Jobs.GET_CATEGORIES + + # Query the API to get all standard categories + categories = self._query_get_all(endpoint) + for category in categories: + # Convert each category dictionary to a StandardCategory object and yield it yield StandardCategory.from_dict(category) diff --git a/sage_desktop_sdk/apis/vendors.py b/sage_desktop_sdk/apis/vendors.py index 40fa4137..cd308dd5 100644 --- a/sage_desktop_sdk/apis/vendors.py +++ b/sage_desktop_sdk/apis/vendors.py @@ -11,20 +11,50 @@ class Vendors(Client): GET_VENDORS = '/AccountsPayable/Api/V1/Vendor.svc/vendors' GET_VENDOR_TYPES = '/AccountsPayable/Api/V1/Vendor.svc/vendors/types' - def get_all(self): + def get_all(self, version: int = None): """ - Get all Vendors - :return: List of Dicts in Vendors Schema + Get all vendors. + + :param version: API version + :type version: int + + :return: A generator yielding vendors in the Vendors Schema + :rtype: generator of Vendor objects """ - vendors = self._query_get_all(Vendors.GET_VENDORS) + if version: + # Append the version query parameter if provided + query_params = '?version={0}'.format(version) + endpoint = Vendors.GET_VENDORS + query_params + else: + endpoint = Vendors.GET_VENDORS + + # Query the API to get all vendors + vendors = self._query_get_all(endpoint) + for vendor in vendors: + # Convert each vendor dictionary to a Vendor object and yield it yield Vendor.from_dict(vendor) - def get_vendor_types(self): + def get_vendor_types(self, version: int = None): """ - Get Vendor Types - :return: List of Dicts in Vendor Types Schema + Get vendor types. + + :param version: API version + :type version: int + + :return: A generator yielding vendor types in the Vendor Types Schema + :rtype: generator of VendorType objects """ - vendor_types = self._query_get_all(Vendors.GET_VENDOR_TYPES) + if version: + # Append the version query parameter if provided + query_params = '?version={0}'.format(version) + endpoint = Vendors.GET_VENDOR_TYPES + query_params + else: + endpoint = Vendors.GET_VENDOR_TYPES + + # Query the API to get all vendor types + vendor_types = self._query_get_all(endpoint) + for vendor_type in vendor_types: + # Convert each vendor type dictionary to a VendorType object and yield it yield VendorType.from_dict(vendor_type) From 806291c3165521a806c9f6b4ff46020e8d37e4be Mon Sep 17 00:00:00 2001 From: Nilesh Pant <58652823+NileshPant1999@users.noreply.github.com> Date: Tue, 7 Nov 2023 17:04:57 +0530 Subject: [PATCH 06/12] fix expense model and relevant migrations (#76) --- apps/fyle/migrations/0001_initial.py | 42 ++++++++--- .../migrations/0002_dependentfieldsetting.py | 37 --------- apps/fyle/models.py | 75 +++++++++++++++++-- sage_desktop_api/models/fields.py | 2 +- 4 files changed, 102 insertions(+), 54 deletions(-) delete mode 100644 apps/fyle/migrations/0002_dependentfieldsetting.py diff --git a/apps/fyle/migrations/0001_initial.py b/apps/fyle/migrations/0001_initial.py index b41dd5d6..3f54c252 100644 --- a/apps/fyle/migrations/0001_initial.py +++ b/apps/fyle/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.1.2 on 2023-10-27 09:30 +# Generated by Django 4.1.2 on 2023-11-07 11:32 import django.contrib.postgres.fields import django.core.validators @@ -27,7 +27,7 @@ class Migration(migrations.Migration): ('values', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=255), help_text='Values for the operator', null=True, size=None)), ('rank', sage_desktop_api.models.fields.IntegerOptionsField(choices=[(1, 1), (2, 2)], default='', help_text='Rank for the filter', null=True)), ('join_by', sage_desktop_api.models.fields.StringOptionsField(choices=[('AND', 'AND'), ('OR', 'OR')], default='', help_text='Used to join the filter (AND/OR)', max_length=3, null=True)), - ('is_custom', sage_desktop_api.models.fields.BooleanFalseField(default=True, help_text='Custom Field or not')), + ('is_custom', sage_desktop_api.models.fields.BooleanFalseField(default=False, help_text='Custom Field or not')), ('custom_field_type', sage_desktop_api.models.fields.StringOptionsField(choices=[('SELECT', 'SELECT'), ('NUMBER', 'NUMBER'), ('TEXT', 'TEXT')], default='', help_text='Custom field type', max_length=255, null=True)), ('workspace', models.ForeignKey(help_text='Reference to Workspace model', on_delete=django.db.models.deletion.PROTECT, to='workspaces.workspace')), ], @@ -50,19 +50,19 @@ class Migration(migrations.Migration): ('org_id', sage_desktop_api.models.fields.StringNullField(help_text='Organization ID', max_length=255, null=True)), ('expense_number', sage_desktop_api.models.fields.StringNotNullField(help_text='Expense Number', max_length=255)), ('claim_number', sage_desktop_api.models.fields.StringNotNullField(help_text='Claim Number', max_length=255)), - ('amount', models.FloatField(help_text='Home Amount')), + ('amount', sage_desktop_api.models.fields.FloatNullField(help_text='Home Amount', null=True)), ('currency', sage_desktop_api.models.fields.StringNotNullField(help_text='Home Currency', max_length=5)), - ('foreign_amount', models.FloatField(help_text='Foreign Amount', null=True)), - ('foreign_currency', sage_desktop_api.models.fields.StringNotNullField(help_text='Foreign Currency', max_length=5)), + ('foreign_amount', sage_desktop_api.models.fields.FloatNullField(help_text='Foreign Amount', null=True)), + ('foreign_currency', sage_desktop_api.models.fields.StringNullField(help_text='Foreign Currency', max_length=5, null=True)), ('settlement_id', sage_desktop_api.models.fields.StringNullField(help_text='Settlement ID', max_length=255, null=True)), - ('reimbursable', sage_desktop_api.models.fields.BooleanFalseField(default=True, help_text='Expense reimbursable or not')), + ('reimbursable', sage_desktop_api.models.fields.BooleanFalseField(default=False, help_text='Expense reimbursable or not')), ('state', sage_desktop_api.models.fields.StringNotNullField(help_text='Expense state', max_length=255)), - ('vendor', sage_desktop_api.models.fields.StringNotNullField(help_text='Vendor', max_length=255)), + ('vendor', sage_desktop_api.models.fields.StringNullField(help_text='Vendor', max_length=255, null=True)), ('cost_center', sage_desktop_api.models.fields.StringNullField(help_text='Fyle Expense Cost Center', max_length=255, null=True)), ('corporate_card_id', sage_desktop_api.models.fields.StringNullField(help_text='Corporate Card ID', max_length=255, null=True)), ('purpose', models.TextField(blank=True, help_text='Purpose', null=True)), ('report_id', sage_desktop_api.models.fields.StringNotNullField(help_text='Report ID', max_length=255)), - ('billable', sage_desktop_api.models.fields.BooleanFalseField(default=True, help_text='Expense billable or not')), + ('billable', sage_desktop_api.models.fields.BooleanFalseField(default=False, help_text='Expense billable or not')), ('file_ids', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=255), help_text='File IDs', null=True, size=None)), ('spent_at', sage_desktop_api.models.fields.CustomDateTimeField(help_text='Expense spent at', null=True)), ('approved_at', sage_desktop_api.models.fields.CustomDateTimeField(help_text='Expense approved at', null=True)), @@ -72,15 +72,37 @@ class Migration(migrations.Migration): ('fund_source', sage_desktop_api.models.fields.StringNotNullField(help_text='Expense fund source', max_length=255)), ('verified_at', sage_desktop_api.models.fields.CustomDateTimeField(help_text='Report verified at', null=True)), ('custom_properties', sage_desktop_api.models.fields.CustomJsonField(default=list, help_text='Custom Properties', null=True)), + ('report_title', models.TextField(blank=True, help_text='Report title', null=True)), + ('payment_number', sage_desktop_api.models.fields.StringNullField(help_text='Expense payment number', max_length=55, null=True)), ('tax_amount', sage_desktop_api.models.fields.FloatNullField(help_text='Tax Amount', null=True)), ('tax_group_id', sage_desktop_api.models.fields.StringNullField(help_text='Tax Group ID', max_length=255, null=True)), - ('exported', sage_desktop_api.models.fields.BooleanFalseField(default=True, help_text='Expense reimbursable or not')), ('previous_export_state', sage_desktop_api.models.fields.StringNullField(help_text='Previous export state', max_length=255, null=True)), ('accounting_export_summary', sage_desktop_api.models.fields.CustomJsonField(default=list, help_text='Accounting Export Summary', null=True)), - ('workspace', models.OneToOneField(help_text='Reference to Workspace model', on_delete=django.db.models.deletion.PROTECT, to='workspaces.workspace')), + ('workspace', models.ForeignKey(help_text='Reference to Workspace model', on_delete=django.db.models.deletion.PROTECT, to='workspaces.workspace')), ], options={ 'db_table': 'expenses', }, ), + migrations.CreateModel( + name='DependentFieldSetting', + fields=[ + ('created_at', models.DateTimeField(auto_now_add=True, help_text='Created at datetime')), + ('updated_at', models.DateTimeField(auto_now=True, help_text='Updated at datetime')), + ('id', models.AutoField(primary_key=True, serialize=False)), + ('is_import_enabled', sage_desktop_api.models.fields.BooleanFalseField(default=False, help_text='Is Import Enabled')), + ('project_field_id', sage_desktop_api.models.fields.IntegerNotNullField(help_text='Fyle Source Field ID')), + ('cost_code_field_name', sage_desktop_api.models.fields.StringNotNullField(help_text='Fyle Cost Code Field Name', max_length=255)), + ('cost_code_field_id', sage_desktop_api.models.fields.StringNotNullField(help_text='Fyle Cost Code Field ID', max_length=255)), + ('cost_code_placeholder', models.TextField(blank=True, help_text='Placeholder for Cost code', null=True)), + ('cost_category_field_name', sage_desktop_api.models.fields.StringNotNullField(help_text='Fyle Cost Category Field Name', max_length=255)), + ('cost_category_field_id', sage_desktop_api.models.fields.StringNotNullField(help_text='Fyle Cost Category Field ID', max_length=255)), + ('cost_category_placeholder', models.TextField(blank=True, help_text='Placeholder for Cost Category', null=True)), + ('last_successful_import_at', sage_desktop_api.models.fields.CustomDateTimeField(help_text='Last Successful Import At', null=True)), + ('workspace', models.OneToOneField(help_text='Reference to Workspace model', on_delete=django.db.models.deletion.PROTECT, to='workspaces.workspace')), + ], + options={ + 'db_table': 'dependent_field_settings', + }, + ), ] diff --git a/apps/fyle/migrations/0002_dependentfieldsetting.py b/apps/fyle/migrations/0002_dependentfieldsetting.py deleted file mode 100644 index d7b010c9..00000000 --- a/apps/fyle/migrations/0002_dependentfieldsetting.py +++ /dev/null @@ -1,37 +0,0 @@ -# Generated by Django 4.1.2 on 2023-11-02 09:29 - -from django.db import migrations, models -import django.db.models.deletion -import sage_desktop_api.models.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('workspaces', '0002_sage300credential_importsetting_fylecredential_and_more'), - ('fyle', '0001_initial'), - ] - - operations = [ - migrations.CreateModel( - name='DependentFieldSetting', - fields=[ - ('created_at', models.DateTimeField(auto_now_add=True, help_text='Created at datetime')), - ('updated_at', models.DateTimeField(auto_now=True, help_text='Updated at datetime')), - ('id', models.AutoField(primary_key=True, serialize=False)), - ('is_import_enabled', sage_desktop_api.models.fields.BooleanFalseField(default=True, help_text='Is Import Enabled')), - ('project_field_id', sage_desktop_api.models.fields.IntegerNotNullField(help_text='Fyle Source Field ID')), - ('cost_code_field_name', sage_desktop_api.models.fields.StringNotNullField(help_text='Fyle Cost Code Field Name', max_length=255)), - ('cost_code_field_id', sage_desktop_api.models.fields.StringNotNullField(help_text='Fyle Cost Code Field ID', max_length=255)), - ('cost_code_placeholder', models.TextField(blank=True, help_text='Placeholder for Cost code', null=True)), - ('cost_category_field_name', sage_desktop_api.models.fields.StringNotNullField(help_text='Fyle Cost Category Field Name', max_length=255)), - ('cost_category_field_id', sage_desktop_api.models.fields.StringNotNullField(help_text='Fyle Cost Category Field ID', max_length=255)), - ('cost_category_placeholder', models.TextField(blank=True, help_text='Placeholder for Cost Category', null=True)), - ('last_successful_import_at', sage_desktop_api.models.fields.CustomDateTimeField(help_text='Last Successful Import At', null=True)), - ('workspace', models.OneToOneField(help_text='Reference to Workspace model', on_delete=django.db.models.deletion.PROTECT, to='workspaces.workspace')), - ], - options={ - 'db_table': 'dependent_field_settings', - }, - ), - ] diff --git a/apps/fyle/models.py b/apps/fyle/models.py index ee7a18e0..b2b384bf 100644 --- a/apps/fyle/models.py +++ b/apps/fyle/models.py @@ -1,3 +1,5 @@ +from typing import List, Dict + from django.db import models from django.contrib.postgres.fields import ArrayField from sage_desktop_api.models.fields import ( @@ -41,6 +43,11 @@ ('not_in', 'not_in') ) +SOURCE_ACCOUNT_MAP = { + 'PERSONAL_CASH_ACCOUNT': 'PERSONAL', + 'PERSONAL_CORPORATE_CREDIT_CARD_ACCOUNT': 'CCC' +} + class ExpenseFilter(BaseForeignWorkspaceModel): """ @@ -59,7 +66,7 @@ class Meta: db_table = 'expense_filters' -class Expense(BaseModel): +class Expense(BaseForeignWorkspaceModel): """ Expense """ @@ -73,14 +80,14 @@ class Expense(BaseModel): org_id = StringNullField(help_text='Organization ID') expense_number = StringNotNullField(help_text='Expense Number') claim_number = StringNotNullField(help_text='Claim Number') - amount = models.FloatField(help_text='Home Amount') + amount = FloatNullField(help_text='Home Amount') currency = StringNotNullField(max_length=5, help_text='Home Currency') - foreign_amount = models.FloatField(null=True, help_text='Foreign Amount') - foreign_currency = StringNotNullField(max_length=5, help_text='Foreign Currency') + foreign_amount = FloatNullField(help_text='Foreign Amount') + foreign_currency = StringNullField(max_length=5, help_text='Foreign Currency') settlement_id = StringNullField(help_text='Settlement ID') reimbursable = BooleanFalseField(help_text='Expense reimbursable or not') state = StringNotNullField(help_text='Expense state') - vendor = StringNotNullField(help_text='Vendor') + vendor = StringNullField(help_text='Vendor') cost_center = StringNullField(help_text='Fyle Expense Cost Center') corporate_card_id = StringNullField(help_text='Corporate Card ID') purpose = models.TextField(null=True, blank=True, help_text='Purpose') @@ -95,15 +102,71 @@ class Expense(BaseModel): fund_source = StringNotNullField(help_text='Expense fund source') verified_at = CustomDateTimeField(help_text='Report verified at') custom_properties = CustomJsonField(help_text="Custom Properties") + report_title = models.TextField(null=True, blank=True, help_text='Report title') + payment_number = StringNullField(max_length=55, help_text='Expense payment number') tax_amount = FloatNullField(help_text='Tax Amount') tax_group_id = StringNullField(help_text='Tax Group ID') - exported = BooleanFalseField(help_text='Expense reimbursable or not') previous_export_state = StringNullField(max_length=255, help_text='Previous export state') accounting_export_summary = CustomJsonField(default=dict, help_text='Accounting Export Summary') class Meta: db_table = 'expenses' + @staticmethod + def create_expense_objects(expenses: List[Dict], workspace_id: int): + """ + Bulk create expense objects + """ + + # Create an empty list to store expense objects + for expense in expenses: + # Iterate through custom property fields and handle empty values + for custom_property_field in expense['custom_properties']: + if expense['custom_properties'][custom_property_field] == '': + expense['custom_properties'][custom_property_field] = None + + # Create or update an Expense object based on expense_id + Expense.objects.update_or_create( + expense_id=expense['id'], + defaults={ + 'employee_email': expense['employee_email'], + 'employee_name': expense['employee_name'], + 'category': expense['category'], + 'sub_category': expense['sub_category'], + 'project': expense['project'], + 'expense_number': expense['expense_number'], + 'org_id': expense['org_id'], + 'claim_number': expense['claim_number'], + 'amount': round(expense['amount'], 2), + 'currency': expense['currency'], + 'foreign_amount': expense['foreign_amount'], + 'foreign_currency': expense['foreign_currency'], + 'tax_amount': expense['tax_amount'], + 'tax_group_id': expense['tax_group_id'], + 'settlement_id': expense['settlement_id'], + 'reimbursable': expense['reimbursable'], + 'billable': expense['billable'] if expense['billable'] else False, + 'state': expense['state'], + 'vendor': expense['vendor'][:250] if expense['vendor'] else None, + 'cost_center': expense['cost_center'], + 'purpose': expense['purpose'], + 'report_id': expense['report_id'], + 'report_title': expense['report_title'], + 'spent_at': expense['spent_at'], + 'approved_at': expense['approved_at'], + 'posted_at': expense['posted_at'], + 'expense_created_at': expense['expense_created_at'], + 'expense_updated_at': expense['expense_updated_at'], + 'fund_source': SOURCE_ACCOUNT_MAP[expense['source_account_type']], + 'verified_at': expense['verified_at'], + 'custom_properties': expense['custom_properties'], + 'payment_number': expense['payment_number'], + 'file_ids': expense['file_ids'], + 'corporate_card_id': expense['corporate_card_id'], + 'workspace_id': workspace_id + } + ) + class DependentFieldSetting(BaseModel): """ diff --git a/sage_desktop_api/models/fields.py b/sage_desktop_api/models/fields.py index 5fc1da46..6fc1d240 100644 --- a/sage_desktop_api/models/fields.py +++ b/sage_desktop_api/models/fields.py @@ -94,7 +94,7 @@ class BooleanFalseField(models.BooleanField): description = "Custom Boolean Field with Default True" def __init__(self, *args, **kwargs): - kwargs['default'] = True # Set the default value to True + kwargs['default'] = False # Set the default value to True super(BooleanFalseField, self).__init__(*args, **kwargs) def toggle(self, instance): From 1f43f988bfa1b96247a2c0af0baea6e4e96ac291 Mon Sep 17 00:00:00 2001 From: Dhaarani <55541808+DhaaraniCIT@users.noreply.github.com> Date: Wed, 8 Nov 2023 12:03:28 +0530 Subject: [PATCH 07/12] cors updation (#78) cors updation --- centralized_run.sh | 2 +- sage_desktop_api/settings.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/centralized_run.sh b/centralized_run.sh index 4b0bb649..4c1c8698 100644 --- a/centralized_run.sh +++ b/centralized_run.sh @@ -7,4 +7,4 @@ python manage.py migrate python manage.py createcachetable --database cache_db # Running development server -gunicorn -c gunicorn_config.py sage_desktop_api.wsgi -b 0.0.0.0:8008 +gunicorn -c gunicorn_config.py sage_desktop_api.wsgi -b 0.0.0.0:8010 diff --git a/sage_desktop_api/settings.py b/sage_desktop_api/settings.py index 1e12e8a1..1e7e15ce 100644 --- a/sage_desktop_api/settings.py +++ b/sage_desktop_api/settings.py @@ -280,3 +280,7 @@ # https://docs.djangoproject.com/en/3.1/howto/static-files/ STATIC_URL = '/static/' + +CORS_ORIGIN_ALLOW_ALL = True + +CORS_ALLOW_HEADERS = ['sentry-trace', 'authorization', 'content-type'] From b24ab7c0d360c03f40760291bd040a8a6fd017a6 Mon Sep 17 00:00:00 2001 From: ruuushhh <66899387+ruuushhh@users.noreply.github.com> Date: Wed, 8 Nov 2023 15:43:52 +0530 Subject: [PATCH 08/12] Bug Fix: Default workspace state set to CONNECTION (#79) --- apps/workspaces/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/workspaces/models.py b/apps/workspaces/models.py index 0e68b610..1fa8bbb8 100644 --- a/apps/workspaces/models.py +++ b/apps/workspaces/models.py @@ -26,7 +26,7 @@ def get_default_onboarding_state(): - return 'EXPORT_SETTINGS' + return 'CONNECTION' class Workspace(models.Model): From 272155eef2538d8848c51883f4be4eee5aa905b1 Mon Sep 17 00:00:00 2001 From: Nilesh Pant <58652823+NileshPant1999@users.noreply.github.com> Date: Thu, 9 Nov 2023 12:57:41 +0530 Subject: [PATCH 09/12] update requirements file and sage300 url (#80) --- apps/workspaces/urls.py | 4 ++-- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/workspaces/urls.py b/apps/workspaces/urls.py index 67318364..f2515d3e 100644 --- a/apps/workspaces/urls.py +++ b/apps/workspaces/urls.py @@ -28,7 +28,7 @@ workspace_app_paths = [ path('', WorkspaceView.as_view(), name='workspaces'), path('ready/', ReadyView.as_view(), name='ready'), - path('/credentials/sage_300/', Sage300CredsView.as_view(), name='sage300-creds'), + path('/credentials/sage300/', Sage300CredsView.as_view(), name='sage300-creds'), path('/export_settings/', ExportSettingView.as_view(), name='export-settings'), path('/import_settings/', ImportSettingView.as_view(), name='import-settings'), path('/advanced_settings/', AdvancedSettingView.as_view(), name='advanced-settings'), @@ -36,7 +36,7 @@ ] other_app_paths = [ - path('/sage_300/', include('apps.sage300.urls')), + path('/sage300/', include('apps.sage300.urls')), path('/fyle/', include('apps.fyle.urls')), path('/accounting_exports/', include('apps.accounting_exports.urls')), path('/mappings/', include('apps.mappings.urls')) diff --git a/requirements.txt b/requirements.txt index e71c44d4..40c5ed75 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ fyle==0.35.0 # Reusable Fyle Packages fyle-rest-auth==1.5.0 -fyle-accounting-mappings==1.27.3 +fyle-accounting-mappings==1.28.0 fyle-integrations-platform-connector==1.36.1 # Postgres Dependincies From 4d241478db559dd6870609e663133882bcfd42ef Mon Sep 17 00:00:00 2001 From: Nilesh Pant <58652823+NileshPant1999@users.noreply.github.com> Date: Fri, 10 Nov 2023 14:22:24 +0530 Subject: [PATCH 10/12] add support for incremental updates (#81) --- apps/mappings/migrations/0001_initial.py | 4 +-- apps/mappings/models.py | 2 +- apps/sage300/helpers.py | 2 +- apps/sage300/models.py | 14 ++++---- apps/sage300/utils.py | 45 +++++++++++++++++++----- 5 files changed, 48 insertions(+), 19 deletions(-) diff --git a/apps/mappings/migrations/0001_initial.py b/apps/mappings/migrations/0001_initial.py index ba650f58..9f2bf16e 100644 --- a/apps/mappings/migrations/0001_initial.py +++ b/apps/mappings/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.1.2 on 2023-11-06 07:01 +# Generated by Django 4.1.2 on 2023-11-09 10:19 from django.db import migrations, models import django.db.models.deletion @@ -24,7 +24,7 @@ class Migration(migrations.Migration): ('job', sage_desktop_api.models.fields.IntegerNullField(help_text='version for job', null=True)), ('standard_category', sage_desktop_api.models.fields.IntegerNullField(help_text='version for standard category', null=True)), ('standard_cost_code', sage_desktop_api.models.fields.IntegerNullField(help_text='version for standard costcode', null=True)), - ('job_category', sage_desktop_api.models.fields.IntegerNullField(help_text='version for job category', null=True)), + ('cost_category', sage_desktop_api.models.fields.IntegerNullField(help_text='version for job category', null=True)), ('cost_code', sage_desktop_api.models.fields.IntegerNullField(help_text='version for costcode', null=True)), ('vendor', sage_desktop_api.models.fields.IntegerNullField(help_text='version for vendor', null=True)), ('commitment', sage_desktop_api.models.fields.IntegerNullField(help_text='version for commitment', null=True)), diff --git a/apps/mappings/models.py b/apps/mappings/models.py index c63e38ca..1c1da730 100644 --- a/apps/mappings/models.py +++ b/apps/mappings/models.py @@ -46,7 +46,7 @@ class Version(BaseModel): job = IntegerNullField(help_text='version for job') standard_category = IntegerNullField(help_text='version for standard category') standard_cost_code = IntegerNullField(help_text='version for standard costcode') - job_category = IntegerNullField(help_text='version for job category') + cost_category = IntegerNullField(help_text='version for job category') cost_code = IntegerNullField(help_text='version for costcode') vendor = IntegerNullField(help_text='version for vendor') commitment = IntegerNullField(help_text='version for commitment') diff --git a/apps/sage300/helpers.py b/apps/sage300/helpers.py index 5e2c3f8a..b0b74ae3 100644 --- a/apps/sage300/helpers.py +++ b/apps/sage300/helpers.py @@ -49,7 +49,7 @@ def sync_dimensions(sage300_credential: Sage300Credential, workspace_id: int) -> sage300_connection = import_string('apps.sage300.utils.SageDesktopConnector')(sage300_credential, workspace_id) # List of dimensions to sync - dimensions = ['categories'] + dimensions = ['accounts', 'vendors', 'commitments', 'jobs', 'standard_categories', 'standard_cost_codes', 'cost_codes', 'cost_categories'] for dimension in dimensions: try: diff --git a/apps/sage300/models.py b/apps/sage300/models.py index 58522e71..f463994d 100644 --- a/apps/sage300/models.py +++ b/apps/sage300/models.py @@ -132,13 +132,13 @@ def bulk_create_or_update(categories_generator: List[Dict], workspace_id: int): record_number_list = [category.id for category in list_of_categories] filters = { - 'category_id__in': record_number_list, + 'cost_category_id__in': record_number_list, 'workspace_id': workspace_id } existing_categories = CostCategory.objects.filter(**filters).values( 'id', - 'category_id', + 'cost_category_id', 'name', 'status' ) @@ -147,8 +147,8 @@ def bulk_create_or_update(categories_generator: List[Dict], workspace_id: int): primary_key_map = {} for existing_category in existing_categories: - existing_cost_type_record_numbers.append(existing_category['category_id']) - primary_key_map[existing_category['category_id']] = { + existing_cost_type_record_numbers.append(existing_category['cost_category_id']) + primary_key_map[existing_category['cost_category_id']] = { 'id': existing_category['id'], 'name': existing_category['name'], 'status': existing_category['status'], @@ -174,7 +174,7 @@ def bulk_create_or_update(categories_generator: List[Dict], workspace_id: int): cost_code_name=cost_code_name, name=category.name, status=category.is_active, - category_id=category.id, + cost_category_id=category.id, workspace_id=workspace_id ) @@ -184,7 +184,7 @@ def bulk_create_or_update(categories_generator: List[Dict], workspace_id: int): elif category.id in primary_key_map.keys() and ( category.name != primary_key_map[category.id]['name'] or category.is_active != primary_key_map[category.id]['status'] ): - category_object.id = primary_key_map[category.id]['category_id'] + category_object.id = primary_key_map[category.id]['cost_category_id'] cost_category_to_be_updated.append(category_object) if cost_category_to_be_created: @@ -194,7 +194,7 @@ def bulk_create_or_update(categories_generator: List[Dict], workspace_id: int): CostCategory.objects.bulk_update( cost_category_to_be_updated, fields=[ 'job_id', 'job_name', 'cost_code_id', 'cost_code_name', - 'name', 'status', 'category_id' + 'name', 'status', 'cost_category_id' ], batch_size=2000 ) diff --git a/apps/sage300/utils.py b/apps/sage300/utils.py index d4cf1b1a..8caaf50f 100644 --- a/apps/sage300/utils.py +++ b/apps/sage300/utils.py @@ -2,6 +2,7 @@ from apps.workspaces.models import Sage300Credential from sage_desktop_sdk.sage_desktop_sdk import SageDesktopSDK from apps.sage300.models import CostCategory +from apps.mappings.models import Version class SageDesktopConnector: @@ -46,6 +47,25 @@ def _create_destination_attribute(self, attribute_type, display_name, value, des 'detail': detail } + def _update_latest_version(self, attribute_type: str): + """ + Update the latest version in Version Table + :param attribute_type: Type of the attribute + """ + + latest_version = DestinationAttribute.objects.filter( + attribute_type=attribute_type + ).order_by('-detail__version').first() + + Version.objects.update_or_create( + workspace_id=self.workspace_id, + defaults={ + attribute_type.lower(): latest_version.detail['version'] + } + ) + + return [] + def _sync_data(self, data, attribute_type, display_name, workspace_id, field_names): """ Synchronize data from Sage Desktop SDK to your application @@ -72,11 +92,14 @@ def _sync_data(self, data, attribute_type, display_name, workspace_id, field_nam DestinationAttribute.bulk_create_or_update_destination_attributes( destination_attributes, attribute_type, workspace_id, True) + self._update_latest_version(attribute_type) + def sync_accounts(self): """ Synchronize accounts from Sage Desktop SDK to your application """ - accounts = self.connection.accounts.get_all() + version = Version.objects.get(workspace_id=self.workspace_id).account + accounts = self.connection.accounts.get_all(version=version) self._sync_data(accounts, 'ACCOUNT', 'accounts', self.workspace_id, ['code', 'version']) return [] @@ -84,7 +107,8 @@ def sync_vendors(self): """ Synchronize vendors from Sage Desktop SDK to your application """ - vendors = self.connection.vendors.get_all() + version = Version.objects.get(workspace_id=self.workspace_id).vendor + vendors = self.connection.vendors.get_all(version=version) field_names = [ 'code', 'version', 'default_expense_account', 'default_standard_category', 'default_standard_costcode', 'type_id', 'created_on_utc' @@ -96,7 +120,8 @@ def sync_jobs(self): """ Synchronize jobs from Sage Desktop SDK to your application """ - jobs = self.connection.jobs.get_all_jobs() + version = Version.objects.get(workspace_id=self.workspace_id).job + jobs = self.connection.jobs.get_all_jobs(version=version) field_names = [ 'code', 'status', 'version', 'account_prefix_id', 'created_on_utc' ] @@ -107,7 +132,8 @@ def sync_standard_cost_codes(self): """ Synchronize standard cost codes from Sage Desktop SDK to your application """ - cost_codes = self.connection.jobs.get_standard_costcodes() + version = Version.objects.get(workspace_id=self.workspace_id).standard_cost_code + cost_codes = self.connection.jobs.get_standard_costcodes(version=version) field_names = ['code', 'version', 'is_standard', 'description'] self._sync_data(cost_codes, 'STANDARD_COST_CODE', 'standard_cost_code', self.workspace_id, field_names) return [] @@ -116,7 +142,8 @@ def sync_standard_categories(self): """ Synchronize standard categories from Sage Desktop SDK to your application """ - categories = self.connection.jobs.get_standard_categories() + version = Version.objects.get(workspace_id=self.workspace_id).standard_category + categories = self.connection.jobs.get_standard_categories(version=version) field_names = ['code', 'version', 'description', 'accumulation_name'] self._sync_data(categories, 'STANDARD_CATEGORY', 'standard_category', self.workspace_id, field_names) return [] @@ -125,7 +152,8 @@ def sync_commitments(self): """ Synchronize commitments from Sage Desktop SDK to your application """ - commitments = self.connection.commitments.get_all() + version = Version.objects.get(workspace_id=self.workspace_id).commitment + commitments = self.connection.commitments.get_all(version=version) field_names = [ 'code', 'is_closed', 'version', 'description', 'is_commited', 'created_on_utc', 'date', 'vendor_id', 'job_id' @@ -137,12 +165,13 @@ def sync_cost_codes(self): """ Synchronize cost codes from Sage Desktop SDK to your application """ - cost_codes = self.connection.cost_codes.get_all_costcodes() + version = Version.objects.get(workspace_id=self.workspace_id).cost_code + cost_codes = self.connection.cost_codes.get_all_costcodes(version=version) field_names = ['code', 'version', 'job_id'] self._sync_data(cost_codes, 'COST_CODE', 'cost_code', self.workspace_id, field_names) return [] - def sync_categories(self): + def sync_cost_categories(self): """ Synchronize categories from Sage Desktop SDK to your application """ From 6ce3a187fe20f297b4b14e762bd496c5caef3dbb Mon Sep 17 00:00:00 2001 From: Nilesh Pant <58652823+NileshPant1999@users.noreply.github.com> Date: Fri, 10 Nov 2023 15:26:56 +0530 Subject: [PATCH 11/12] PR1: add support for accounting export and expense creations (#72) --- apps/accounting_exports/models.py | 97 +++++++++++++++- apps/fyle/exceptions.py | 37 ++++++ apps/fyle/models.py | 13 ++- apps/fyle/queue.py | 60 ++++++++++ apps/fyle/tasks.py | 91 +++++++++++++++ apps/fyle/urls.py | 20 +++- apps/fyle/views.py | 18 +++ apps/mappings/imports/tasks.py | 2 +- ...ve_advancedsetting_schedule_id_and_more.py | 24 ++++ apps/workspaces/models.py | 3 +- apps/workspaces/tasks.py | 109 ++++++++++++++++++ tests/test_workspaces/test_views.py | 3 - 12 files changed, 465 insertions(+), 12 deletions(-) create mode 100644 apps/fyle/exceptions.py create mode 100644 apps/fyle/queue.py create mode 100644 apps/fyle/tasks.py create mode 100644 apps/workspaces/migrations/0003_remove_advancedsetting_schedule_id_and_more.py create mode 100644 apps/workspaces/tasks.py diff --git a/apps/accounting_exports/models.py b/apps/accounting_exports/models.py index 3c703bee..eaae2791 100644 --- a/apps/accounting_exports/models.py +++ b/apps/accounting_exports/models.py @@ -1,5 +1,8 @@ +from typing import List from django.db import models from django.contrib.postgres.fields import ArrayField +from django.contrib.postgres.aggregates import ArrayAgg +from django.db.models import Count from fyle_accounting_mappings.models import ExpenseAttribute @@ -13,9 +16,10 @@ StringOptionsField, IntegerNullField ) -from apps.workspaces.models import BaseForeignWorkspaceModel, BaseModel +from apps.workspaces.models import BaseForeignWorkspaceModel, BaseModel, ExportSetting from apps.fyle.models import Expense + TYPE_CHOICES = ( ('INVOICES', 'INVOICES'), ('DIRECT_COST', 'DIRECT_COST'), @@ -31,6 +35,56 @@ ) +def _group_expenses(expenses: List[Expense], export_setting: ExportSetting, fund_source: str): + """ + Group expenses based on specified fields + """ + + credit_card_expense_grouped_by = export_setting.credit_card_expense_grouped_by + credit_card_expense_date = export_setting.credit_card_expense_date + reimbursable_expense_grouped_by = export_setting.reimbursable_expense_grouped_by + reimbursable_expense_date = export_setting.reimbursable_expense_date + + default_fields = ['employee_email', 'fund_source'] + report_grouping_fields = ['report_id', 'claim_number'] + expense_grouping_fields = ['expense_id', 'expense_number'] + + # Define a mapping for fund sources and their associated group fields + fund_source_mapping = { + 'CCC': { + 'group_by': report_grouping_fields if credit_card_expense_grouped_by == 'REPORT' else expense_grouping_fields, + 'date_field': credit_card_expense_date.lower() if credit_card_expense_date != 'LAST_SPENT_AT' else None + }, + 'PERSONAL': { + 'group_by': report_grouping_fields if reimbursable_expense_grouped_by == 'REPORT' else expense_grouping_fields, + 'date_field': reimbursable_expense_date.lower() if reimbursable_expense_date != 'LAST_SPENT_AT' else None + } + } + + # Update expense_group_fields based on the fund_source + fund_source_data = fund_source_mapping.get(fund_source) + group_by_field = fund_source_data.get('group_by') + date_field = fund_source_data.get('date_field') + + default_fields.extend([group_by_field, fund_source]) + + if date_field: + default_fields.append(date_field) + + # Extract expense IDs from the provided expenses + expense_ids = [expense.id for expense in expenses] + + # Retrieve expenses from the database + expenses = Expense.objects.filter(id__in=expense_ids).all() + + # Create expense groups by grouping expenses based on specified fields + expense_groups = list(expenses.values(*default_fields).annotate( + total=Count('*'), expense_ids=ArrayAgg('id')) + ) + + return expense_groups + + class AccountingExport(BaseForeignWorkspaceModel): """ Table to store accounting exports @@ -50,6 +104,47 @@ class AccountingExport(BaseForeignWorkspaceModel): class Meta: db_table = 'accounting_exports' + @staticmethod + def create_accounting_export(expense_objects: List[Expense], fund_source: str, workspace_id): + """ + Group expenses by report_id and fund_source, format date fields, and create AccountingExport objects. + """ + + # Retrieve the ExportSetting for the workspace + export_setting = ExportSetting.objects.get(workspace_id=workspace_id) + + # Group expenses based on specified fields and fund_source + accounting_exports = _group_expenses(expense_objects, export_setting, fund_source) + + fund_source_map = { + 'PERSONAL': 'reimbursable', + 'CCC': 'credit_card' + } + + for accounting_export in accounting_exports: + # Determine the date field based on fund_source + date_field = getattr(export_setting, f"{fund_source_map.get(fund_source)}_expense_date", None) + + # Calculate and assign 'last_spent_at' based on the chosen date field + if date_field == 'last_spent_at': + latest_expense = Expense.objects.filter(id__in=accounting_export['expense_ids']).order_by('-spent_at').first() + accounting_export['last_spent_at'] = latest_expense.spent_at if latest_expense else None + + # Store expense IDs and remove unnecessary keys + expense_ids = accounting_export['expense_ids'] + accounting_export.pop('total') + accounting_export.pop('expense_ids') + + # Create an AccountingExport object for the expense group + accounting_export_instance = AccountingExport.objects.create( + workspace_id=workspace_id, + fund_source=accounting_export['fund_source'], + description=accounting_export, + ) + + # Add related expenses to the AccountingExport object + accounting_export_instance.expenses.add(*expense_ids) + class Error(BaseForeignWorkspaceModel): """ diff --git a/apps/fyle/exceptions.py b/apps/fyle/exceptions.py new file mode 100644 index 00000000..4d14ae44 --- /dev/null +++ b/apps/fyle/exceptions.py @@ -0,0 +1,37 @@ +import logging +import traceback +from functools import wraps + +from fyle.platform.exceptions import NoPrivilegeError + +from apps.workspaces.models import FyleCredential + +logger = logging.getLogger(__name__) +logger.level = logging.INFO + + +def handle_exceptions(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except FyleCredential.DoesNotExist: + logger.info('Fyle credentials not found %s', args[1]) # args[1] is workspace_id + args[2].detail = {'message': 'Fyle credentials do not exist in workspace'} + args[2].status = 'FAILED' + args[2].save() + + except NoPrivilegeError: + logger.info('Invalid Fyle Credentials / Admin is disabled') + args[2].detail = {'message': 'Invalid Fyle Credentials / Admin is disabled'} + args[2].status = 'FAILED' + args[2].save() + + except Exception: + error = traceback.format_exc() + args[2].detail = {'error': error} + args[2].status = 'FATAL' + args[2].save() + logger.exception('Something unexpected happened workspace_id: %s %s', args[1], args[2].detail) + + return wrapper diff --git a/apps/fyle/models.py b/apps/fyle/models.py index b2b384bf..293967fd 100644 --- a/apps/fyle/models.py +++ b/apps/fyle/models.py @@ -2,6 +2,7 @@ from django.db import models from django.contrib.postgres.fields import ArrayField + from sage_desktop_api.models.fields import ( StringNotNullField, StringNullField, @@ -12,7 +13,7 @@ CustomDateTimeField, CustomEmailField, FloatNullField, - IntegerNotNullField, + IntegerNotNullField ) from apps.workspaces.models import BaseModel, BaseForeignWorkspaceModel @@ -119,6 +120,8 @@ def create_expense_objects(expenses: List[Dict], workspace_id: int): """ # Create an empty list to store expense objects + expense_objects = [] + for expense in expenses: # Iterate through custom property fields and handle empty values for custom_property_field in expense['custom_properties']: @@ -126,7 +129,7 @@ def create_expense_objects(expenses: List[Dict], workspace_id: int): expense['custom_properties'][custom_property_field] = None # Create or update an Expense object based on expense_id - Expense.objects.update_or_create( + expense_object, _ = Expense.objects.update_or_create( expense_id=expense['id'], defaults={ 'employee_email': expense['employee_email'], @@ -167,6 +170,12 @@ def create_expense_objects(expenses: List[Dict], workspace_id: int): } ) + # Check if an AccountingExport related to the expense object already exists + if not Expense.objects.filter(accountingexport__isnull=False).distinct(): + expense_objects.append(expense_object) + + return expense_objects + class DependentFieldSetting(BaseModel): """ diff --git a/apps/fyle/queue.py b/apps/fyle/queue.py new file mode 100644 index 00000000..a182b515 --- /dev/null +++ b/apps/fyle/queue.py @@ -0,0 +1,60 @@ +""" +All the tasks which are queued into django-q + * User Triggered Async Tasks + * Schedule Triggered Async Tasks +""" +from django_q.tasks import async_task + +from apps.fyle.tasks import ( + import_credit_card_expenses, + import_reimbursable_expenses +) +from apps.accounting_exports.models import AccountingExport + + +def queue_import_reimbursable_expenses(workspace_id: int, synchronous: bool = False): + """ + Queue Import of Reimbursable Expenses from Fyle + :param workspace_id: Workspace id + :return: None + """ + accounting_export, _ = AccountingExport.objects.update_or_create( + workspace_id=workspace_id, + type='FETCHING_REIMBURSABLE_EXPENSES', + defaults={ + 'status': 'IN_PROGRESS' + } + ) + + if not synchronous: + async_task( + 'apps.fyle.tasks.import_reimbursable_expenses', + workspace_id, accounting_export, + ) + return + + import_reimbursable_expenses(workspace_id, accounting_export) + + +def queue_import_credit_card_expenses(workspace_id: int, synchronous: bool = False): + """ + Queue Import of Credit Card Expenses from Fyle + :param workspace_id: Workspace id + :return: None + """ + accounting_export, _ = AccountingExport.objects.update_or_create( + workspace_id=workspace_id, + type='FETCHING_CREDIT_CARD_EXPENSES', + defaults={ + 'status': 'IN_PROGRESS' + } + ) + + if not synchronous: + async_task( + 'apps.fyle.tasks.import_credit_card_expenses', + workspace_id, accounting_export, + ) + return + + import_credit_card_expenses(workspace_id, accounting_export) diff --git a/apps/fyle/tasks.py b/apps/fyle/tasks.py new file mode 100644 index 00000000..33d4c041 --- /dev/null +++ b/apps/fyle/tasks.py @@ -0,0 +1,91 @@ +""" +All Tasks from which involve Fyle APIs + +1. Import Reimbursable Expenses from Fyle +2. Import Credit Card Expenses from Fyle +""" +import logging +from datetime import datetime +from django.db import transaction + +from fyle_integrations_platform_connector import PlatformConnector + +from apps.accounting_exports.models import AccountingExport +from apps.workspaces.models import ExportSetting, Workspace, FyleCredential +from apps.fyle.models import Expense +from apps.fyle.exceptions import handle_exceptions + +SOURCE_ACCOUNT_MAP = { + 'PERSONAL': 'PERSONAL_CASH_ACCOUNT', + 'CCC': 'PERSONAL_CORPORATE_CREDIT_CARD_ACCOUNT' +} + +logger = logging.getLogger(__name__) +logger.level = logging.INFO + + +def import_expenses(workspace_id, accounting_export: AccountingExport, source_account_type, fund_source_key): + """ + Common logic for importing expenses from Fyle + :param accounting_export: Task log object + :param workspace_id: workspace id + :param source_account_type: Fyle source account type + :param fund_source_key: Key for accessing fund source specific fields in ExportSetting + """ + + fund_source_map = { + 'PERSONAL': 'reimbursable', + 'CCC': 'credit_card' + } + export_settings = ExportSetting.objects.get(workspace_id=workspace_id) + workspace = Workspace.objects.get(pk=workspace_id) + last_synced_at = getattr(workspace, f"{fund_source_key.lower()}_last_synced_at", None) + fyle_credentials = FyleCredential.objects.get(workspace_id=workspace_id) + + platform = PlatformConnector(fyle_credentials) + + expenses = platform.expenses.get( + source_account_type=[source_account_type], + state=getattr(export_settings, f"{fund_source_key.lower()}_expense_state"), + settled_at=last_synced_at if getattr(export_settings, f"{fund_source_map.get(fund_source_key)}_expense_state") == 'PAYMENT_PROCESSING' else None, + approved_at=last_synced_at if getattr(export_settings, f"{fund_source_map.get(fund_source_key)}_expense_state") == 'APPROVED' else None, + filter_credit_expenses=(fund_source_key == 'CCC'), + last_paid_at=last_synced_at if getattr(export_settings, f"{fund_source_key.lower()}_expense_state") == 'PAID' else None + ) + + if expenses: + setattr(workspace, f"{fund_source_map.get(fund_source_key)}_last_synced_at", datetime.now()) + workspace.save() + + with transaction.atomic(): + expenses_object = Expense.create_expense_objects(expenses, workspace_id) + AccountingExport.create_accounting_export_report_id( + expenses_object, + fund_source=fund_source_key, + workspace_id=workspace_id + ) + + accounting_export.status = 'COMPLETE' + accounting_export.errors = None + + accounting_export.save() + + +@handle_exceptions +def import_reimbursable_expenses(workspace_id, accounting_export: AccountingExport): + """ + Import reimbursable expenses from Fyle + :param accounting_export: Accounting Export object + :param workspace_id: workspace id + """ + import_expenses(workspace_id, accounting_export, 'PERSONAL_CASH_ACCOUNT', 'PERSONAL') + + +@handle_exceptions +def import_credit_card_expenses(workspace_id, accounting_export: AccountingExport): + """ + Import credit card expenses from Fyle + :param accounting_export: AccountingExport object + :param workspace_id: workspace id + """ + import_expenses(workspace_id, accounting_export, 'PERSONAL_CORPORATE_CREDIT_CARD_ACCOUNT', 'CCC') diff --git a/apps/fyle/urls.py b/apps/fyle/urls.py index 6de9ff9d..f40a032b 100644 --- a/apps/fyle/urls.py +++ b/apps/fyle/urls.py @@ -15,6 +15,8 @@ """ from django.urls import path +import itertools + from apps.fyle.views import ( ImportFyleAttributesView, ExpenseFilterView, @@ -22,16 +24,26 @@ CustomFieldView, FyleFieldsView, DependentFieldSettingView, - ExportableExpenseGroupsView + ExportableExpenseGroupsView, + AccoutingExportSyncView ) -urlpatterns = [ - path('import_attributes/', ImportFyleAttributesView.as_view(), name='import-fyle-attributes'), +accounting_exports_path = [ + path('exportable_accounting_exports/', ExportableExpenseGroupsView.as_view(), name='exportable-accounting-exports'), + path('expense_groups/sync/', AccoutingExportSyncView.as_view(), name='sync-accounting-exports'), +] + +other_paths = [ path('expense_filters//', ExpenseFilterDeleteView.as_view(), name='expense-filters'), path('expense_filters/', ExpenseFilterView.as_view(), name='expense-filters'), path('expense_fields/', CustomFieldView.as_view(), name='fyle-expense-fields'), path('fields/', FyleFieldsView.as_view(), name='fyle-fields'), path('dependent_field_settings/', DependentFieldSettingView.as_view(), name='dependent-field'), - path('exportable_accounting_exports/', ExportableExpenseGroupsView.as_view(), name='exportable-accounting-exports'), ] + +fyle_dimension_paths = [ + path('import_attributes/', ImportFyleAttributesView.as_view(), name='import-fyle-attributes') +] + +urlpatterns = list(itertools.chain(accounting_exports_path, fyle_dimension_paths, other_paths)) diff --git a/apps/fyle/views.py b/apps/fyle/views.py index 5153f414..40280d0b 100644 --- a/apps/fyle/views.py +++ b/apps/fyle/views.py @@ -14,6 +14,7 @@ ) from apps.fyle.models import ExpenseFilter, DependentFieldSetting from apps.fyle.helpers import get_exportable_accounting_exports_ids +from apps.fyle.queue import queue_import_reimbursable_expenses, queue_import_credit_card_expenses logger = logging.getLogger(__name__) @@ -86,3 +87,20 @@ def get(self, request, *args, **kwargs): data={'exportable_expense_group_ids': exportable_ids}, status=status.HTTP_200_OK ) + + +class AccoutingExportSyncView(generics.CreateAPIView): + """ + Create expense groups + """ + def post(self, request, *args, **kwargs): + """ + Post expense groups creation + """ + + queue_import_reimbursable_expenses(kwargs['workspace_id'], synchronous=True) + queue_import_credit_card_expenses(kwargs['workspace_id'], synchronous=True) + + return Response( + status=status.HTTP_200_OK + ) diff --git a/apps/mappings/imports/tasks.py b/apps/mappings/imports/tasks.py index b949b17f..bcfc35a7 100644 --- a/apps/mappings/imports/tasks.py +++ b/apps/mappings/imports/tasks.py @@ -11,7 +11,7 @@ SOURCE_FIELD_CLASS_MAP = { 'CATEGORY': Category, - 'PROJECT': Project + 'PROJECT': Project, } diff --git a/apps/workspaces/migrations/0003_remove_advancedsetting_schedule_id_and_more.py b/apps/workspaces/migrations/0003_remove_advancedsetting_schedule_id_and_more.py new file mode 100644 index 00000000..ddfeb43c --- /dev/null +++ b/apps/workspaces/migrations/0003_remove_advancedsetting_schedule_id_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 4.1.2 on 2023-11-07 11:11 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_q', '0014_schedule_cluster'), + ('workspaces', '0002_sage300credential_importsetting_fylecredential_and_more'), + ] + + operations = [ + migrations.RemoveField( + model_name='advancedsetting', + name='schedule_id', + ), + migrations.AddField( + model_name='advancedsetting', + name='schedule', + field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.PROTECT, to='django_q.schedule'), + ), + ] diff --git a/apps/workspaces/models.py b/apps/workspaces/models.py index 1fa8bbb8..653b9bb1 100644 --- a/apps/workspaces/models.py +++ b/apps/workspaces/models.py @@ -1,6 +1,7 @@ from django.db import models from django.contrib.auth import get_user_model from django.contrib.postgres.fields import ArrayField +from django_q.models import Schedule from sage_desktop_api.models.fields import ( StringNotNullField, @@ -212,11 +213,11 @@ class AdvancedSetting(BaseModel): ) schedule_is_enabled = BooleanFalseField(help_text='Boolean to check if schedule is enabled') schedule_start_datetime = CustomDateTimeField(help_text='Schedule start date and time') - schedule_id = StringNullField(help_text='Schedule id') interval_hours = IntegerNullField(help_text='Interval in hours') emails_selected = CustomJsonField(help_text='Emails Selected For Email Notification') emails_added = CustomJsonField(help_text='Emails Selected For Email Notification') auto_create_vendor = BooleanFalseField(help_text='Auto create vendor') + schedule = models.OneToOneField(Schedule, on_delete=models.PROTECT, null=True) class Meta: db_table = 'advanced_settings' diff --git a/apps/workspaces/tasks.py b/apps/workspaces/tasks.py new file mode 100644 index 00000000..27cd7937 --- /dev/null +++ b/apps/workspaces/tasks.py @@ -0,0 +1,109 @@ +from typing import List +import logging +from datetime import datetime, timedelta +from django_q.models import Schedule + +from apps.workspaces.models import ExportSetting, AdvancedSetting +from apps.accounting_exports.models import AccountingExport, AccountingExportSummary + + +logger = logging.getLogger(__name__) + + +def run_import_export(workspace_id: int, export_mode = None): + """ + Run process to export to sage300 + + :param workspace_id: Workspace id + """ + + export_settings = ExportSetting.objects.get(workspace_id=workspace_id) + advance_settings = AdvancedSetting.objects.get(workspace_id=workspace_id) + accounting_summary = AccountingExportSummary.objects.get(workspace_id=workspace_id) + + last_exported_at = datetime.now() + is_expenses_exported = False + + # For Reimbursable Expenses + if export_settings.reimbursable_expenses_export_type: + accounting_export = AccountingExport.objects.get( + workspace_id=workspace_id, + type='FETCHING_REIMBURSABLE_EXPENSES' + ) + + if accounting_export.status == 'COMPLETE': + accounting_export_ids = AccountingExport.objects.filter( + fund_source='PERSONAL', exported_at__isnull=True).values_list('id', flat=True) + + if len(accounting_export_ids): + is_expenses_exported = True + + """ + Export Logic goes here + """ + + # For Credit Card Expenses + if export_settings.credit_card_expense_export_type: + accounting_export = AccountingExport.objects.get( + workspace_id=workspace_id, + type='FETCHING_CREDIT_CARD_EXPENENSES' + ) + + if accounting_export.status == 'COMPLETE': + accounting_export_ids = AccountingExport.objects.filter( + fund_source='CCC', exported_at__isnull=True).values_list('id', flat=True) + + if len(accounting_export_ids): + is_expenses_exported = True + + """ + Export Logic goes here + """ + + if is_expenses_exported: + accounting_summary.last_exported_at = last_exported_at + accounting_summary.export_mode = export_mode or 'MANUAL' + + if advance_settings: + accounting_summary.next_export_at = last_exported_at + timedelta(hours=24) + + accounting_summary.save() + + +def schedule_sync(workspace_id: int, schedule_enabled: bool, hours: int, email_added: List, emails_selected: List): + + advance_settings = AdvancedSetting.objects.get(workspace_id=workspace_id) + + if advance_settings.schedule_is_enabled: + advance_settings.schedule_is_enabled = schedule_enabled + advance_settings.start_datetime = datetime.now() + advance_settings.interval_hours = hours + advance_settings.emails_selected = emails_selected + + if email_added: + advance_settings.emails_added = email_added + + # create next run by adding hours to current time + next_run = datetime.now() + timedelta(hours=hours) + + schedule, _ = Schedule.objects.update_or_create( + func='apps.workspaces.tasks.run_import_export', + args='{}'.format(workspace_id), + defaults={ + 'schedule_type': Schedule.MINUTES, + 'minutes': hours * 60, + 'next_run': next_run + } + ) + + advance_settings.schedule = schedule + advance_settings.save() + + elif not schedule_enabled and advance_settings.schedule_is_enabled: + schedule = advance_settings.schedule + advance_settings.enabled = schedule_enabled + advance_settings.schedule = None + advance_settings.save() + schedule.delete() + + return advance_settings diff --git a/tests/test_workspaces/test_views.py b/tests/test_workspaces/test_views.py index c3691153..a4c7eacb 100644 --- a/tests/test_workspaces/test_views.py +++ b/tests/test_workspaces/test_views.py @@ -264,7 +264,6 @@ def test_advanced_settings(api_client, test_connection): 'expense_link' ] assert response.data['schedule_is_enabled'] is False - assert response.data['schedule_id'] is None assert response.data['emails_selected'] == [ { 'name': 'Shwetabh Kumar', @@ -288,7 +287,6 @@ def test_advanced_settings(api_client, test_connection): 'expense_link' ] assert response.data['schedule_is_enabled'] is False - assert response.data['schedule_id'] is None assert response.data['emails_selected'] == [ { 'name': 'Shwetabh Kumar', @@ -314,7 +312,6 @@ def test_advanced_settings(api_client, test_connection): 'report_number' ] assert response.data['schedule_is_enabled'] is False - assert response.data['schedule_id'] is None assert response.data['emails_selected'] == [ { 'name': 'Shwetabh Kumar', From 1c6f0b3cf8bc5ce98b55613d85dea9670a939f8f Mon Sep 17 00:00:00 2001 From: ruuushhh <66899387+ruuushhh@users.noreply.github.com> Date: Tue, 14 Nov 2023 17:08:26 +0530 Subject: [PATCH 12/12] Import Standard Cost Code, Standard Category as Cost Center (#66) --- apps/mappings/imports/modules/base.py | 2 + apps/mappings/imports/modules/cost_centers.py | 57 +++++++++++++++++++ apps/mappings/imports/tasks.py | 2 + tests/conftest.py | 30 ++++++++++ .../test_imports/test_modules/fixtures.py | 15 +++++ .../test_modules/test_cost_centers.py | 21 +++++++ 6 files changed, 127 insertions(+) create mode 100644 apps/mappings/imports/modules/cost_centers.py create mode 100644 tests/test_mappings/test_imports/test_modules/test_cost_centers.py diff --git a/apps/mappings/imports/modules/base.py b/apps/mappings/imports/modules/base.py index f503ff41..d21d8d68 100644 --- a/apps/mappings/imports/modules/base.py +++ b/apps/mappings/imports/modules/base.py @@ -187,6 +187,8 @@ def construct_payload_and_import_to_fyle( """ Construct Payload and Import to fyle in Batches """ + is_auto_sync_status_allowed = self.get_auto_sync_permission() + filters = self.construct_attributes_filter(self.destination_field) destination_attributes_count = DestinationAttribute.objects.filter(**filters).count() diff --git a/apps/mappings/imports/modules/cost_centers.py b/apps/mappings/imports/modules/cost_centers.py new file mode 100644 index 00000000..18a7d29b --- /dev/null +++ b/apps/mappings/imports/modules/cost_centers.py @@ -0,0 +1,57 @@ +from datetime import datetime +from typing import List +from apps.mappings.imports.modules.base import Base +from fyle_accounting_mappings.models import DestinationAttribute + + +class CostCenter(Base): + """ + Class for Cost Center module + """ + + def __init__(self, workspace_id: int, destination_field: str, sync_after: datetime): + super().__init__( + workspace_id=workspace_id, + source_field="COST_CENTER", + destination_field=destination_field, + platform_class_name="cost_centers", + sync_after=sync_after, + ) + + def trigger_import(self): + """ + Trigger import for Cost Center module + """ + self.check_import_log_and_start_import() + + def construct_fyle_payload( + self, + paginated_destination_attributes: List[DestinationAttribute], + existing_fyle_attributes_map: object, + is_auto_sync_status_allowed: bool + ): + """ + Construct Fyle payload for CostCenter module + :param paginated_destination_attributes: List of paginated destination attributes + :param existing_fyle_attributes_map: Existing Fyle attributes map + :param is_auto_sync_status_allowed: Is auto sync status allowed + :return: Fyle payload + """ + payload = [] + + for attribute in paginated_destination_attributes: + cost_center = { + 'name': attribute.value, + 'code': attribute.destination_id, + 'is_enabled': True if attribute.active is None else attribute.active, + 'description': 'Cost Center - {0}, Id - {1}'.format( + attribute.value, + attribute.destination_id + ) + } + + # Create a new cost-center if it does not exist in Fyle + if attribute.value.lower() not in existing_fyle_attributes_map: + payload.append(cost_center) + + return payload diff --git a/apps/mappings/imports/tasks.py b/apps/mappings/imports/tasks.py index bcfc35a7..9a490797 100644 --- a/apps/mappings/imports/tasks.py +++ b/apps/mappings/imports/tasks.py @@ -5,6 +5,7 @@ from apps.mappings.models import ImportLog from apps.mappings.imports.modules.categories import Category from apps.mappings.imports.modules.projects import Project +from apps.mappings.imports.modules.cost_centers import CostCenter from apps.mappings.imports.modules.expense_custom_fields import ExpenseCustomField from apps.fyle.models import DependentFieldSetting @@ -12,6 +13,7 @@ SOURCE_FIELD_CLASS_MAP = { 'CATEGORY': Category, 'PROJECT': Project, + 'COST_CENTER': CostCenter } diff --git a/tests/conftest.py b/tests/conftest.py index 854bf6e7..ac3a87ba 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -309,6 +309,36 @@ def add_project_mappings(): ) +@pytest.fixture() +@pytest.mark.django_db(databases=['default']) +def add_cost_center_mappings(): + """ + Pytest fixtue to add cost center mappings to a workspace + """ + workspace_ids = [ + 1, 2, 3 + ] + for workspace_id in workspace_ids: + DestinationAttribute.objects.create( + workspace_id=workspace_id, + attribute_type='COST_CENTER', + display_name='Direct Mail Campaign', + value='Direct Mail Campaign', + destination_id='10064', + detail='Cost Center - Direct Mail Campaign, Id - 10064', + active=True + ) + DestinationAttribute.objects.create( + workspace_id=workspace_id, + attribute_type='COST_CENTER', + display_name='Platform APIs', + value='Platform APIs', + destination_id='10081', + detail='Cost Center - Platform APIs, Id - 10081', + active=True + ) + + @pytest.fixture() @pytest.mark.django_db(databases=['default']) def add_export_settings(): diff --git a/tests/test_mappings/test_imports/test_modules/fixtures.py b/tests/test_mappings/test_imports/test_modules/fixtures.py index 4f83520c..7315a2bc 100644 --- a/tests/test_mappings/test_imports/test_modules/fixtures.py +++ b/tests/test_mappings/test_imports/test_modules/fixtures.py @@ -29,5 +29,20 @@ 'is_enabled': False, 'name': 'Platform APIs' } + ], + "create_fyle_cost_center_payload_create_new_case": + [ + { + 'code': '10064', + 'description': 'Cost Center - Direct Mail Campaign, Id - 10064', + 'is_enabled': True, + 'name': 'Direct Mail Campaign' + }, + { + 'description': 'Cost Center - Platform APIs, Id - 10081', + 'is_enabled': True, + 'name': 'Platform APIs', + 'code': '10081', + } ] } diff --git a/tests/test_mappings/test_imports/test_modules/test_cost_centers.py b/tests/test_mappings/test_imports/test_modules/test_cost_centers.py new file mode 100644 index 00000000..1a1dbed3 --- /dev/null +++ b/tests/test_mappings/test_imports/test_modules/test_cost_centers.py @@ -0,0 +1,21 @@ +from apps.mappings.imports.modules.cost_centers import CostCenter +from fyle_accounting_mappings.models import DestinationAttribute +from .fixtures import data + + +def test_construct_fyle_payload(api_client, test_connection, mocker, create_temp_workspace, add_sage300_creds, add_fyle_credentials, add_cost_center_mappings): + cost_center = CostCenter(1, 'COST_CENTER', None) + + # create new case + paginated_destination_attributes = DestinationAttribute.objects.filter(workspace_id=1, attribute_type='COST_CENTER') + + existing_fyle_attributes_map = {} + is_auto_sync_status_allowed = cost_center.get_auto_sync_permission() + + fyle_payload = cost_center.construct_fyle_payload( + paginated_destination_attributes, + existing_fyle_attributes_map, + is_auto_sync_status_allowed + ) + + assert fyle_payload == data['create_fyle_cost_center_payload_create_new_case']