Refactor Dependent Fields Import (#304)

* Adding new Cost Types Model and Test Script

* fix table name

* sync with generator and new method

* almost there, rough version

* new table, new api for dependent fields

* bump version

* almost there . . .

* move dependent fields to separate file

* incremental fetches, sync, create/update done

* Fix exports

* fix comments

* add sleep and remove cost type from sync

* fix broken test and update fixture db

* add tests for dependent fields

* increase coverage

* more more tests

---------

Co-authored-by: ashwin1111 <[email protected]>

.gitignore

@@ -161,3 +161,4 @@ docker-compose.yml
 
 # Cache db
 cache.db
+fyle_integrations_platform_connector/

apps/fyle/__init__.py

@@ -0,0 +1 @@
+default_app_config = 'apps.fyle.apps.FyleConfig'

apps/fyle/apps.py

@@ -2,4 +2,8 @@
 
 
 class FyleConfig(AppConfig):
-    name = 'fyle'
+    name = 'apps.fyle'
+
+    def ready(self):
+        super(FyleConfig, self).ready()
+        import apps.fyle.signals

apps/fyle/helpers.py

@@ -221,4 +221,9 @@ def construct_expense_filter_query(expense_filters: List[ExpenseFilter]):
         # Set the join type for the additonal filter
         join_by = expense_filter.join_by
 
-    return final_filter
+    return final_filter
+
+def connect_to_platform(workspace_id: int) -> PlatformConnector:
+    fyle_credentials: FyleCredential = FyleCredential.objects.get(workspace_id=workspace_id)
+
+    return PlatformConnector(fyle_credentials=fyle_credentials)

apps/fyle/migrations/0020_dependentfield.py

@@ -0,0 +1,34 @@
+# Generated by Django 3.1.14 on 2023-06-13 19:39
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('workspaces', '0025_auto_20230417_1124'),
+        ('fyle', '0019_expense_report_title'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='DependentField',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('is_import_enabled', models.BooleanField(help_text='Is Import Enabled')),
+                ('project_field_id', models.IntegerField(help_text='Fyle Source Field ID')),
+                ('cost_code_field_name', models.CharField(help_text='Fyle Cost Code Field Name', max_length=255)),
+                ('cost_code_field_id', models.IntegerField(help_text='Fyle Cost Code Field ID')),
+                ('cost_type_field_name', models.CharField(help_text='Fyle Cost Type Field Name', max_length=255)),
+                ('cost_type_field_id', models.IntegerField(help_text='Fyle Cost Type Field ID')),
+                ('last_successful_import_at', models.DateTimeField(help_text='Last Successful Import At', null=True)),
+                ('created_at', models.DateTimeField(auto_now_add=True, help_text='Created at')),
+                ('updated_at', models.DateTimeField(auto_now=True, help_text='Updated at')),
+                ('workspace', models.OneToOneField(help_text='Reference to Workspace', on_delete=django.db.models.deletion.PROTECT, to='workspaces.workspace')),
+            ],
+            options={
+                'db_table': 'dependent_fields',
+            },
+        ),
+    ]

apps/fyle/migrations/0021_auto_20230615_0808.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.1.14 on 2023-06-15 08:08
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('workspaces', '0025_auto_20230417_1124'),
+        ('fyle', '0020_dependentfield'),
+    ]
+
+    operations = [
+        migrations.RenameModel(
+            old_name='DependentField',
+            new_name='DependentFieldSetting',
+        ),
+        migrations.AlterModelTable(
+            name='dependentfieldsetting',
+            table='dependent_field_settings',
+        ),
+    ]

apps/fyle/models.py

@@ -510,4 +510,25 @@ class ExpenseFilter(models.Model):
     updated_at = models.DateTimeField(auto_now=True, help_text='Updated at')
 
     class Meta:
-        db_table = 'expense_filters'
+        db_table = 'expense_filters'
+
+
+class DependentFieldSetting(models.Model):
+    """
+    Fyle Dependent Fields
+    DB Table: dependent_field_settings:
+    """
+    id = models.AutoField(primary_key=True)
+    is_import_enabled = models.BooleanField(help_text='Is Import Enabled')
+    project_field_id = models.IntegerField(help_text='Fyle Source Field ID')
+    cost_code_field_name = models.CharField(max_length=255, help_text='Fyle Cost Code Field Name')
+    cost_code_field_id = models.IntegerField(help_text='Fyle Cost Code Field ID')
+    cost_type_field_name = models.CharField(max_length=255, help_text='Fyle Cost Type Field Name')
+    cost_type_field_id = models.IntegerField(help_text='Fyle Cost Type Field ID')
+    workspace = models.OneToOneField(Workspace, on_delete=models.PROTECT, help_text='Reference to Workspace')
+    last_successful_import_at = models.DateTimeField(null=True, help_text='Last Successful Import At')
+    created_at = models.DateTimeField(auto_now_add=True, help_text='Created at')
+    updated_at = models.DateTimeField(auto_now=True, help_text='Updated at')
+
+    class Meta:
+        db_table = 'dependent_field_settings'

apps/fyle/serializers.py

@@ -2,7 +2,7 @@
 
 from fyle_accounting_mappings.models import ExpenseAttribute
 
-from .models import Expense, ExpenseFilter, ExpenseGroup, ExpenseGroupSettings, Reimbursement
+from .models import Expense, ExpenseFilter, ExpenseGroup, ExpenseGroupSettings, DependentFieldSetting
 
 
 class ExpenseGroupSerializer(serializers.ModelSerializer):
@@ -72,4 +72,17 @@ def create(self, validated_data):
             defaults=validated_data
         )
 
-        return expense_filter
+        return expense_filter
+
+
+class DependentFieldSettingSerializer(serializers.ModelSerializer):
+    """
+    Dependent Field serializer
+    """
+    project_field_id = serializers.IntegerField(required=False)
+    cost_code_field_id = serializers.IntegerField(required=False)
+    cost_type_field_id = serializers.IntegerField(required=False)
+
+    class Meta:
+        model = DependentFieldSetting
+        fields = '__all__'

apps/fyle/signals.py

@@ -0,0 +1,59 @@
+"""
+Fyle Signal
+"""
+import logging
+
+from django.db.models.signals import post_save, pre_save
+from django.dispatch import receiver
+
+from apps.sage_intacct.dependent_fields import create_dependent_custom_field_in_fyle
+from apps.sage_intacct.dependent_fields import schedule_dependent_field_imports
+
+from .helpers import connect_to_platform
+from .models import DependentFieldSetting
+
+
+logger = logging.getLogger(__name__)
+logger.level = logging.INFO
+
+
+@receiver(pre_save, sender=DependentFieldSetting)
+def run_pre_save_dependent_field_settings_triggers(sender, instance: DependentFieldSetting, **kwargs):
+    """
+    :param sender: Sender Class
+    :param instance: Row instance of Sender Class
+    :return: None
+    """
+    # Patch alert - Skip creating dependent fields if they're already created
+    if instance.cost_code_field_id:
+        return
+
+    platform = connect_to_platform(instance.workspace_id)
+
+    instance.project_field_id = platform.expense_fields.get_project_field_id()
+
+    cost_code = create_dependent_custom_field_in_fyle(
+        workspace_id=instance.workspace_id,
+        fyle_attribute_type=instance.cost_code_field_name,
+        platform=platform,
+        parent_field_id=instance.project_field_id
+    )
+    instance.cost_code_field_id = cost_code['data']['id']
+
+    cost_type = create_dependent_custom_field_in_fyle(
+        workspace_id=instance.workspace_id,
+        fyle_attribute_type=instance.cost_type_field_name,
+        platform=platform,
+        parent_field_id=instance.cost_code_field_id
+    )
+    instance.cost_type_field_id = cost_type['data']['id']
+
+
+@receiver(post_save, sender=DependentFieldSetting)
+def run_post_save_dependent_field_settings_triggers(sender, instance: DependentFieldSetting, **kwargs):
+    """
+    :param sender: Sender Class
+    :param instance: Row instance of Sender Class
+    :return: None
+    """
+    schedule_dependent_field_imports(instance.workspace_id, instance.is_import_enabled)

apps/fyle/urls.py

@@ -16,10 +16,14 @@
 from django.urls import path
 import itertools
 
-from .views import CustomFieldView, ExpenseFilterView, ExpenseGroupExpenseView, ExpenseGroupView, ExpenseGroupScheduleView, ExpenseGroupByIdView, \
-    ExpenseView, EmployeeView, CategoryView, ProjectView, CostCenterView, FyleFieldsView, \
-        ExpenseAttributesView, ExpenseGroupSettingsView, RefreshFyleDimensionView, SyncFyleDimensionView, \
-        ExpenseGroupCountView
+from .views import (
+    CustomFieldView, ExpenseFilterView, ExpenseGroupExpenseView,
+    ExpenseGroupView, ExpenseGroupScheduleView, ExpenseGroupByIdView,
+    ExpenseView, EmployeeView, CategoryView, ProjectView, CostCenterView,
+    FyleFieldsView, ExpenseAttributesView, ExpenseGroupSettingsView,
+    RefreshFyleDimensionView, SyncFyleDimensionView, ExpenseGroupCountView,
+    DependentFieldSettingView
+)
 
 expense_groups_paths = [
     path('expense_groups/', ExpenseGroupView.as_view()),
@@ -44,7 +48,8 @@
     path('fyle_fields/', FyleFieldsView.as_view()),
     path('expense_filters/', ExpenseFilterView.as_view(), name='expense-filters'),
     path('expenses/', ExpenseView.as_view(), name='expenses'),
-    path('custom_fields/', CustomFieldView.as_view(), name='custom-field')
+    path('custom_fields/', CustomFieldView.as_view(), name='custom-field'),
+    path('dependent_field_settings/', DependentFieldSettingView.as_view(), name='dependent-field')
 ]
 
 urlpatterns = list(itertools.chain(expense_groups_paths, fyle_dimension_paths, other_paths))

apps/fyle/views.py

@@ -19,9 +19,12 @@
 
 from .tasks import create_expense_groups, schedule_expense_group_creation
 from .helpers import check_interval_and_sync_dimension, sync_dimensions
-from .models import Expense, ExpenseFilter, ExpenseGroup, ExpenseGroupSettings
-from .serializers import ExpenseFilterSerializer, ExpenseGroupExpenseSerializer, ExpenseGroupSerializer, ExpenseSerializer, ExpenseFieldSerializer, \
-    ExpenseGroupSettingsSerializer
+from .models import Expense, ExpenseFilter, ExpenseGroup, ExpenseGroupSettings, DependentFieldSetting
+from .serializers import (
+    ExpenseFilterSerializer, ExpenseGroupExpenseSerializer, ExpenseGroupSerializer,
+    ExpenseSerializer, ExpenseFieldSerializer, ExpenseGroupSettingsSerializer,
+    DependentFieldSettingSerializer
+)
 
 
 class ExpenseGroupView(generics.ListCreateAPIView):
@@ -432,3 +435,11 @@ def get(self, request, *args, **kwargs):
             status=status.HTTP_200_OK
         )
 
+
+class DependentFieldSettingView(generics.CreateAPIView, generics.RetrieveUpdateAPIView):
+    """
+    Dependent Field view
+    """
+    serializer_class = DependentFieldSettingSerializer
+    lookup_field = 'workspace_id'
+    queryset = DependentFieldSetting.objects.all()

apps/mappings/signals.py

@@ -14,7 +14,7 @@
 from fyle.platform.exceptions import WrongParamsError
 
 from apps.mappings.tasks import schedule_cost_centers_creation, schedule_fyle_attributes_creation,\
-    upload_attributes_to_fyle, upload_dependent_field_to_fyle
+    upload_attributes_to_fyle
 from apps.workspaces.models import Configuration
 from apps.mappings.helpers import schedule_or_delete_fyle_import_tasks
 from apps.workspaces.tasks import delete_cards_mapping_settings
@@ -39,7 +39,7 @@ def run_post_mapping_settings_triggers(sender, instance: MappingSetting, **kwarg
 
     if instance.is_custom:
         schedule_fyle_attributes_creation(int(instance.workspace_id))
-    
+
     if configuration:
         delete_cards_mapping_settings(configuration)
 
@@ -54,19 +54,16 @@ def run_pre_mapping_settings_triggers(sender, instance: MappingSetting, **kwargs
     default_attributes = ['EMPLOYEE', 'CATEGORY', 'PROJECT', 'COST_CENTER', 'TAX_GROUP', 'CORPORATE_CARD']
 
     instance.source_field = instance.source_field.upper().replace(' ', '_')
-    parent_field_id = instance.expense_field.source_field_id if instance.expense_field else None
 
     if instance.source_field not in default_attributes:
-        #TODO: sync intacct fields before we upload custom field
+        # TODO: sync intacct fields before we upload custom field
         try:
-            if not instance.expense_field:
-                upload_attributes_to_fyle(
-                    workspace_id=int(instance.workspace_id),
-                    sageintacct_attribute_type=instance.destination_field,
-                    fyle_attribute_type=instance.source_field,
-                    parent_field_id=parent_field_id,
-                    source_placeholder=instance.source_placeholder
-                )
+            upload_attributes_to_fyle(
+                workspace_id=int(instance.workspace_id),
+                sageintacct_attribute_type=instance.destination_field,
+                fyle_attribute_type=instance.source_field,
+                source_placeholder=instance.source_placeholder
+            )
 
         except WrongParamsError as error:
             logger.error(
@@ -87,6 +84,5 @@ def run_pre_mapping_settings_triggers(sender, instance: MappingSetting, **kwargs
             'apps.mappings.tasks.auto_create_expense_fields_mappings',
             int(instance.workspace_id),
             instance.destination_field,
-            instance.source_field,
-            parent_field_id
+            instance.source_field
         )