Code prepend support for PROJECT, Dep Fields, CATEGORY, MERCHANT, COS…

…T_CENTER, CUSTOM fields (#205)

* add import_code_fields, test cases

* fix param name

* add condition before adding dep fields

* add condition before adding dep fields

* Job, Dep Field code-prepending support (#206)

* Job, Dep Field code-prepending support

* auto map feature

* fix comments - add util func, handle code in VENDOR_TYPE

* fix existing test cases

* Add check for syncing jobs, deps and add test cases

* update func name

* added unit tests

* fix failing test

* add project import related test cases

* fix lint

* remove redundant db calls

* rename helper func, fix is_job_sync_allowed method

* add support for code prepending in CATEGORY (#208)

* add support for code prepending in CATEGORY

* rename the helper method

* rename the helper method

* add support for code prepending in MERCHANT (#209)

* add support for code prepending in MERCHANT

* change the callback methods are sent

* bug fix

* rename the helper method

* fix callback method mapping of vendor

* Code naming support cost center (#210)

* add support for code prepending in COST_CENTER

* rename the helper method

* add support for code prepending in CUSTOM attribute (#211)

* add support for code prepending in CUSTOM attribute

* improve test case

* add test cases (#212)

* Job, Dep Field code-prepending support

* auto map feature

* fix comments - add util func, handle code in VENDOR_TYPE

* fix existing test cases

* Add check for syncing jobs, deps and add test cases

* update func name

* added unit tests

* fix failing test

* add project import related test cases

* fix lint

* remove redundant db calls

* add support for code prepending in CATEGORY

* add support for code prepending in MERCHANT

* change the callback methods are sent

* bug fix

* add support for code prepending in COST_CENTER

* rename helper func, fix is_job_sync_allowed method

* rename the helper method

* rename the helper method

* rename the helper method

* fix callback method mapping of vendor

* rename the helper method

* add support for code prepending in CUSTOM attribute

* improve test case

* add test cases

* method refactor

* add validation for adding code post import (#214)

* add validation for adding code post import

* add loggers

* remove mapping, refactor validations (#215)

* remove mapping, refactor validations

* fix lint

* Add code_prepend support in export (#216)

* Add code_prepend support in export

* code-prepend changes in custom fields

* fix 500 internal error (#217)

* remove unnecessary check

* optimize the api call

* optimize the query

* add api for import config (#218)

* refactor project callback func, dep field func

* add additional condition for updating the dependent field setting last import flag

* bump accounting mapping version

* remove print stmt, remove mapping condition

* add delimiter in the imported field

.github/workflows/codecov.yml

@@ -15,9 +15,9 @@ jobs:
       - uses: actions/checkout@v2 
       - name: Bring up Services and Run Tests
         run: |
-          docker-compose -f docker-compose-pipeline.yml build
-          docker-compose -f docker-compose-pipeline.yml up -d
-          docker-compose -f docker-compose-pipeline.yml exec -T api pytest tests/ --cov --junit-xml=test-reports/report.xml --cov-report=xml --cov-fail-under=70          
+          docker compose -f docker-compose-pipeline.yml build
+          docker compose -f docker-compose-pipeline.yml up -d
+          docker compose -f docker-compose-pipeline.yml exec -T api pytest tests/ --cov --junit-xml=test-reports/report.xml --cov-report=xml --cov-fail-under=70          
           echo "STATUS=$(cat pytest-coverage.txt | grep 'Required test' | awk '{ print $1 }')" >> $GITHUB_ENV
           echo "FAILED=$(cat test-reports/report.xml | awk -F'=' '{print $5}' | awk -F' ' '{gsub(/"/, "", $1); print $1}')" >> $GITHUB_ENV
       - name: Upload coverage reports to Codecov with GitHub Action

.github/workflows/tests.yml

@@ -12,9 +12,9 @@ jobs:
       - uses: actions/checkout@v2 
       - name: Bring up Services and Run Tests
         run: |
-          docker-compose -f docker-compose-pipeline.yml build
-          docker-compose -f docker-compose-pipeline.yml up -d
-          docker-compose -f docker-compose-pipeline.yml exec -T api pytest tests/ --cov --junit-xml=test-reports/report.xml --cov-report=xml --cov-fail-under=70          
+          docker compose -f docker-compose-pipeline.yml build
+          docker compose -f docker-compose-pipeline.yml up -d
+          docker compose -f docker-compose-pipeline.yml exec -T api pytest tests/ --cov --junit-xml=test-reports/report.xml --cov-report=xml --cov-fail-under=70          
           echo "STATUS=$(cat pytest-coverage.txt | grep 'Required test' | awk '{ print $1 }')" >> $GITHUB_ENV
           echo "FAILED=$(cat test-reports/report.xml | awk -F'=' '{print $5}' | awk -F' ' '{gsub(/"/, "", $1); print $1}')" >> $GITHUB_ENV
       - name: Upload coverage reports to Codecov with GitHub Action

apps/mappings/helpers.py

@@ -0,0 +1,28 @@
+from datetime import datetime, timedelta, timezone
+from apps.mappings.models import ImportLog
+
+
+def prepend_code_to_name(prepend_code_in_name: bool, value: str, code: str = None) -> str:
+    """
+    Format the attribute name based on the use_code_in_naming flag
+    """
+    if prepend_code_in_name and code:
+        return "{}: {}".format(code, value)
+    return value
+
+
+def is_job_sync_allowed(import_log: ImportLog = None) -> bool:
+    """
+    Check if job sync is allowed
+    """
+    time_difference = datetime.now(timezone.utc) - timedelta(minutes=30)
+    time_difference = time_difference.replace(tzinfo=timezone.utc)
+
+    if (
+        not import_log
+        or import_log.last_successful_run_at is None
+        or import_log.last_successful_run_at < time_difference
+    ):
+        return True
+
+    return False

apps/mappings/imports/modules/base.py

@@ -19,7 +19,7 @@
 from apps.sage300.utils import SageDesktopConnector
 from apps.mappings.exceptions import handle_import_exceptions
 from apps.accounting_exports.models import Error
-
+from apps.mappings.helpers import prepend_code_to_name
 
 logger = logging.getLogger(__name__)
 logger.level = logging.INFO
@@ -36,12 +36,14 @@ def __init__(
         destination_field: str,
         platform_class_name: str,
         sync_after:datetime,
+        use_code_in_naming: bool = False
     ):
         self.workspace_id = workspace_id
         self.source_field = source_field
         self.destination_field = destination_field
         self.platform_class_name = platform_class_name
         self.sync_after = sync_after
+        self.use_code_in_naming = use_code_in_naming
 
     def get_platform_class(self, platform: PlatformConnector):
         """
@@ -92,7 +94,11 @@ def remove_duplicate_attributes(self, destination_attributes: List[DestinationAt
         attribute_values = []
 
         for destination_attribute in destination_attributes:
-            if destination_attribute.value.lower() not in attribute_values:
+            attribute_value = destination_attribute.value
+            attribute_value = prepend_code_to_name(self.use_code_in_naming, destination_attribute.value, destination_attribute.code)
+
+            if attribute_value.lower() not in attribute_values:
+                destination_attribute.value = attribute_value
                 unique_attributes.append(destination_attribute)
                 attribute_values.append(destination_attribute.value.lower())
 
@@ -165,8 +171,7 @@ def create_mappings(self):
         destination_attributes_without_duplicates = []
         destination_attributes = DestinationAttribute.objects.filter(
             workspace_id=self.workspace_id,
-            attribute_type=self.destination_field,
-            mapping__isnull=True
+            attribute_type=self.destination_field
         ).order_by('value', 'id')
         destination_attributes_without_duplicates = self.remove_duplicate_attributes(destination_attributes)
         if destination_attributes_without_duplicates:

apps/mappings/imports/modules/categories.py

@@ -1,21 +1,29 @@
+import logging
 from datetime import datetime
-from typing import List
+from typing import List, Dict
+from apps.workspaces.models import ImportSetting, FyleCredential
 from apps.mappings.imports.modules.base import Base
-from fyle_accounting_mappings.models import DestinationAttribute, CategoryMapping
+from apps.mappings.helpers import prepend_code_to_name
+from fyle_integrations_platform_connector import PlatformConnector
+from fyle_accounting_mappings.models import DestinationAttribute, ExpenseAttribute, CategoryMapping
+
+logger = logging.getLogger(__name__)
+logger.level = logging.INFO
 
 
 class Category(Base):
     """
     Class for Category module
     """
 
-    def __init__(self, workspace_id: int, destination_field: str, sync_after: datetime):
+    def __init__(self, workspace_id: int, destination_field: str, sync_after: datetime, use_code_in_naming: bool = False):
         super().__init__(
             workspace_id=workspace_id,
             source_field="CATEGORY",
             destination_field=destination_field,
             platform_class_name="categories",
             sync_after=sync_after,
+            use_code_in_naming=use_code_in_naming
         )
 
     def trigger_import(self):
@@ -62,8 +70,7 @@ def create_mappings(self):
         """
         filters = {
             "workspace_id": self.workspace_id,
-            "attribute_type": self.destination_field,
-            "destination_account__isnull": True
+            "attribute_type": self.destination_field
         }
 
         # get all the destination attributes that have category mappings as null
@@ -81,3 +88,63 @@ def create_mappings(self):
             self.destination_field,
             self.workspace_id,
         )
+
+
+def disable_categories(workspace_id: int, categories_to_disable: Dict, *args, **kwargs):
+    """
+    categories_to_disable object format:
+    {
+        'destination_id': {
+            'value': 'old_category_name',
+            'updated_value': 'new_category_name',
+            'code': 'old_code',
+            'update_code': 'new_code' ---- if the code is updated else same as code
+        }
+    }
+    """
+    fyle_credentials = FyleCredential.objects.get(workspace_id=workspace_id)
+    platform = PlatformConnector(fyle_credentials=fyle_credentials)
+
+    use_code_in_naming = ImportSetting.objects.filter(workspace_id=workspace_id, import_code_fields__contains=['ACCOUNT']).first()
+
+    category_values = []
+    for category_map in categories_to_disable.values():
+        category_name = prepend_code_to_name(prepend_code_in_name=use_code_in_naming, value=category_map['value'], code=category_map['code'])
+        category_values.append(category_name)
+
+    filters = {
+        'workspace_id': workspace_id,
+        'attribute_type': 'CATEGORY',
+        'value__in': category_values,
+        'active': True
+    }
+
+    # Expense attribute value map is as follows: {old_category_name: destination_id}
+    expense_attribute_value_map = {}
+    for k, v in categories_to_disable.items():
+        category_name = prepend_code_to_name(prepend_code_in_name=use_code_in_naming, value=v['value'], code=v['code'])
+        expense_attribute_value_map[category_name] = k
+
+    expense_attributes = ExpenseAttribute.objects.filter(**filters)
+
+    bulk_payload = []
+    for expense_attribute in expense_attributes:
+        code = expense_attribute_value_map.get(expense_attribute.value, None)
+        if code:
+            payload = {
+                'name': expense_attribute.value,
+                'code': code,
+                'is_enabled': False,
+                'id': expense_attribute.source_id
+            }
+            bulk_payload.append(payload)
+        else:
+            logger.error(f"Category not found in categories_to_disable: {expense_attribute.value}")
+
+    if bulk_payload:
+        logger.info(f"Disabling Category in Fyle | WORKSPACE_ID: {workspace_id} | COUNT: {len(bulk_payload)}")
+        platform.categories.post_bulk(bulk_payload)
+    else:
+        logger.info(f"No Categories to Disable in Fyle | WORKSPACE_ID: {workspace_id}")
+
+    return bulk_payload

apps/mappings/imports/modules/cost_centers.py

@@ -1,21 +1,29 @@
+import logging
 from datetime import datetime
-from typing import List
+from typing import List, Dict
 from apps.mappings.imports.modules.base import Base
-from fyle_accounting_mappings.models import DestinationAttribute
+from fyle_accounting_mappings.models import DestinationAttribute, ExpenseAttribute, MappingSetting
+from apps.workspaces.models import FyleCredential, ImportSetting
+from fyle_integrations_platform_connector import PlatformConnector
+from apps.mappings.helpers import prepend_code_to_name
+
+logger = logging.getLogger(__name__)
+logger.level = logging.INFO
 
 
 class CostCenter(Base):
     """
     Class for Cost Center module
     """
 
-    def __init__(self, workspace_id: int, destination_field: str, sync_after: datetime):
+    def __init__(self, workspace_id: int, destination_field: str, sync_after: datetime, use_code_in_naming: bool = False):
         super().__init__(
             workspace_id=workspace_id,
             source_field="COST_CENTER",
             destination_field=destination_field,
             platform_class_name="cost_centers",
             sync_after=sync_after,
+            use_code_in_naming=use_code_in_naming
         )
 
     def trigger_import(self):
@@ -55,3 +63,67 @@ def construct_fyle_payload(
                 payload.append(cost_center)
 
         return payload
+
+
+def disable_cost_centers(workspace_id: int, cost_centers_to_disable: Dict, *args, **kwargs):
+    """
+    cost_centers_to_disable object format:
+    {
+        'destination_id': {
+            'value': 'old_cost_center_name',
+            'updated_value': 'new_cost_center_name',
+            'code': 'old_code',
+            'update_code': 'new_code' ---- if the code is updated else same as code
+        }
+    }
+    """
+    destination_type = MappingSetting.objects.get(workspace_id=workspace_id, source_field='COST_CENTER').destination_field
+    use_code_in_naming = ImportSetting.objects.filter(workspace_id=workspace_id, import_code_fields__contains=[destination_type]).first()
+
+    fyle_credentials = FyleCredential.objects.get(workspace_id=workspace_id)
+    platform = PlatformConnector(fyle_credentials=fyle_credentials)
+
+    cost_center_values = []
+    for cost_center_map in cost_centers_to_disable.values():
+        cost_center_name = prepend_code_to_name(prepend_code_in_name=use_code_in_naming, value=cost_center_map['value'], code=cost_center_map['code'])
+        cost_center_values.append(cost_center_name)
+
+    filters = {
+        'workspace_id': workspace_id,
+        'attribute_type': 'COST_CENTER',
+        'value__in': cost_center_values,
+        'active': True
+    }
+
+    expense_attribute_value_map = {}
+    for k, v in cost_centers_to_disable.items():
+        cost_center_name = prepend_code_to_name(prepend_code_in_name=use_code_in_naming, value=v['value'], code=v['code'])
+        expense_attribute_value_map[cost_center_name] = k
+
+    expense_attributes = ExpenseAttribute.objects.filter(**filters)
+
+    bulk_payload = []
+    for expense_attribute in expense_attributes:
+        code = expense_attribute_value_map.get(expense_attribute.value, None)
+        if code:
+            payload = {
+                'name': expense_attribute.value,
+                'code': code,
+                'is_enabled': False,
+                'id': expense_attribute.source_id,
+                'description': 'Cost Center - {0}, Id - {1}'.format(
+                    expense_attribute.value,
+                    code
+                )
+            }
+            bulk_payload.append(payload)
+        else:
+            logger.error(f"Cost Center with value {expense_attribute.value} not found | WORKSPACE_ID: {workspace_id}")
+
+    if bulk_payload:
+        logger.info(f"Disabling Cost Center in Fyle | WORKSPACE_ID: {workspace_id} | COUNT: {len(bulk_payload)}")
+        platform.cost_centers.post_bulk(bulk_payload)
+    else:
+        logger.info(f"No Cost Center to Disable in Fyle | WORKSPACE_ID: {workspace_id}")
+
+    return bulk_payload

apps/mappings/imports/modules/expense_custom_fields.py

@@ -1,3 +1,4 @@
+import logging
 from datetime import datetime
 from typing import List, Dict
 from apps.mappings.imports.modules.base import Base
@@ -11,18 +12,22 @@
 from apps.workspaces.models import FyleCredential
 from apps.mappings.constants import FYLE_EXPENSE_SYSTEM_FIELDS
 
+logger = logging.getLogger(__name__)
+logger.level = logging.INFO
+
 
 class ExpenseCustomField(Base):
     """
     Class for ExepenseCustomField module
     """
-    def __init__(self, workspace_id: int, source_field: str, destination_field: str, sync_after: datetime):
+    def __init__(self, workspace_id: int, source_field: str, destination_field: str, sync_after: datetime, use_code_in_naming: bool = False):
         super().__init__(
             workspace_id=workspace_id,
             source_field=source_field,
             destination_field=destination_field,
             platform_class_name='expense_custom_fields',
-            sync_after=sync_after
+            sync_after=sync_after,
+            use_code_in_naming=use_code_in_naming
         )
 
     def trigger_import(self):