add support for incremental updates (#81)

apps/mappings/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.1.2 on 2023-11-06 07:01
+# Generated by Django 4.1.2 on 2023-11-09 10:19
 
 from django.db import migrations, models
 import django.db.models.deletion
@@ -24,7 +24,7 @@ class Migration(migrations.Migration):
                 ('job', sage_desktop_api.models.fields.IntegerNullField(help_text='version for job', null=True)),
                 ('standard_category', sage_desktop_api.models.fields.IntegerNullField(help_text='version for standard category', null=True)),
                 ('standard_cost_code', sage_desktop_api.models.fields.IntegerNullField(help_text='version for standard costcode', null=True)),
-                ('job_category', sage_desktop_api.models.fields.IntegerNullField(help_text='version for job category', null=True)),
+                ('cost_category', sage_desktop_api.models.fields.IntegerNullField(help_text='version for job category', null=True)),
                 ('cost_code', sage_desktop_api.models.fields.IntegerNullField(help_text='version for costcode', null=True)),
                 ('vendor', sage_desktop_api.models.fields.IntegerNullField(help_text='version for vendor', null=True)),
                 ('commitment', sage_desktop_api.models.fields.IntegerNullField(help_text='version for commitment', null=True)),

apps/mappings/models.py

@@ -46,7 +46,7 @@ class Version(BaseModel):
     job = IntegerNullField(help_text='version for job')
     standard_category = IntegerNullField(help_text='version for standard category')
     standard_cost_code = IntegerNullField(help_text='version for standard costcode')
-    job_category = IntegerNullField(help_text='version for job category')
+    cost_category = IntegerNullField(help_text='version for job category')
     cost_code = IntegerNullField(help_text='version for costcode')
     vendor = IntegerNullField(help_text='version for vendor')
     commitment = IntegerNullField(help_text='version for commitment')

apps/sage300/helpers.py

@@ -49,7 +49,7 @@ def sync_dimensions(sage300_credential: Sage300Credential, workspace_id: int) ->
     sage300_connection = import_string('apps.sage300.utils.SageDesktopConnector')(sage300_credential, workspace_id)
 
     # List of dimensions to sync
-    dimensions = ['categories']
+    dimensions = ['accounts', 'vendors', 'commitments', 'jobs', 'standard_categories', 'standard_cost_codes',  'cost_codes', 'cost_categories']
 
     for dimension in dimensions:
         try:

apps/sage300/models.py

@@ -132,13 +132,13 @@ def bulk_create_or_update(categories_generator: List[Dict], workspace_id: int):
         record_number_list = [category.id for category in list_of_categories]
 
         filters = {
-            'category_id__in': record_number_list,
+            'cost_category_id__in': record_number_list,
             'workspace_id': workspace_id
         }
 
         existing_categories = CostCategory.objects.filter(**filters).values(
             'id',
-            'category_id',
+            'cost_category_id',
             'name',
             'status'
         )
@@ -147,8 +147,8 @@ def bulk_create_or_update(categories_generator: List[Dict], workspace_id: int):
         primary_key_map = {}
 
         for existing_category in existing_categories:
-            existing_cost_type_record_numbers.append(existing_category['category_id'])
-            primary_key_map[existing_category['category_id']] = {
+            existing_cost_type_record_numbers.append(existing_category['cost_category_id'])
+            primary_key_map[existing_category['cost_category_id']] = {
                 'id': existing_category['id'],
                 'name': existing_category['name'],
                 'status': existing_category['status'],
@@ -174,7 +174,7 @@ def bulk_create_or_update(categories_generator: List[Dict], workspace_id: int):
                 cost_code_name=cost_code_name,
                 name=category.name,
                 status=category.is_active,
-                category_id=category.id,
+                cost_category_id=category.id,
                 workspace_id=workspace_id
             )
 
@@ -184,7 +184,7 @@ def bulk_create_or_update(categories_generator: List[Dict], workspace_id: int):
             elif category.id in primary_key_map.keys() and (
                 category.name != primary_key_map[category.id]['name'] or category.is_active != primary_key_map[category.id]['status']
             ):
-                category_object.id = primary_key_map[category.id]['category_id']
+                category_object.id = primary_key_map[category.id]['cost_category_id']
                 cost_category_to_be_updated.append(category_object)
 
         if cost_category_to_be_created:
@@ -194,7 +194,7 @@ def bulk_create_or_update(categories_generator: List[Dict], workspace_id: int):
             CostCategory.objects.bulk_update(
                 cost_category_to_be_updated, fields=[
                     'job_id', 'job_name', 'cost_code_id', 'cost_code_name',
-                    'name', 'status', 'category_id'
+                    'name', 'status', 'cost_category_id'
                 ],
                 batch_size=2000
             )

apps/sage300/utils.py

@@ -2,6 +2,7 @@
 from apps.workspaces.models import Sage300Credential
 from sage_desktop_sdk.sage_desktop_sdk import SageDesktopSDK
 from apps.sage300.models import CostCategory
+from apps.mappings.models import Version
 
 
 class SageDesktopConnector:
@@ -46,6 +47,25 @@ def _create_destination_attribute(self, attribute_type, display_name, value, des
             'detail': detail
         }
 
+    def _update_latest_version(self, attribute_type: str):
+        """
+        Update the latest version in Version Table
+        :param attribute_type: Type of the attribute
+        """
+
+        latest_version = DestinationAttribute.objects.filter(
+            attribute_type=attribute_type
+        ).order_by('-detail__version').first()
+
+        Version.objects.update_or_create(
+            workspace_id=self.workspace_id,
+            defaults={
+                attribute_type.lower(): latest_version.detail['version']
+            }
+        )
+
+        return []
+
     def _sync_data(self, data, attribute_type, display_name, workspace_id, field_names):
         """
         Synchronize data from Sage Desktop SDK to your application
@@ -72,19 +92,23 @@ def _sync_data(self, data, attribute_type, display_name, workspace_id, field_nam
         DestinationAttribute.bulk_create_or_update_destination_attributes(
             destination_attributes, attribute_type, workspace_id, True)
 
+        self._update_latest_version(attribute_type)
+
     def sync_accounts(self):
         """
         Synchronize accounts from Sage Desktop SDK to your application
         """
-        accounts = self.connection.accounts.get_all()
+        version = Version.objects.get(workspace_id=self.workspace_id).account
+        accounts = self.connection.accounts.get_all(version=version)
         self._sync_data(accounts, 'ACCOUNT', 'accounts', self.workspace_id, ['code', 'version'])
         return []
 
     def sync_vendors(self):
         """
         Synchronize vendors from Sage Desktop SDK to your application
         """
-        vendors = self.connection.vendors.get_all()
+        version = Version.objects.get(workspace_id=self.workspace_id).vendor
+        vendors = self.connection.vendors.get_all(version=version)
         field_names = [
             'code', 'version', 'default_expense_account', 'default_standard_category',
             'default_standard_costcode', 'type_id', 'created_on_utc'
@@ -96,7 +120,8 @@ def sync_jobs(self):
         """
         Synchronize jobs from Sage Desktop SDK to your application
         """
-        jobs = self.connection.jobs.get_all_jobs()
+        version = Version.objects.get(workspace_id=self.workspace_id).job
+        jobs = self.connection.jobs.get_all_jobs(version=version)
         field_names = [
             'code', 'status', 'version', 'account_prefix_id', 'created_on_utc'
         ]
@@ -107,7 +132,8 @@ def sync_standard_cost_codes(self):
         """
         Synchronize standard cost codes from Sage Desktop SDK to your application
         """
-        cost_codes = self.connection.jobs.get_standard_costcodes()
+        version = Version.objects.get(workspace_id=self.workspace_id).standard_cost_code
+        cost_codes = self.connection.jobs.get_standard_costcodes(version=version)
         field_names = ['code', 'version', 'is_standard', 'description']
         self._sync_data(cost_codes, 'STANDARD_COST_CODE', 'standard_cost_code', self.workspace_id, field_names)
         return []
@@ -116,7 +142,8 @@ def sync_standard_categories(self):
         """
         Synchronize standard categories from Sage Desktop SDK to your application
         """
-        categories = self.connection.jobs.get_standard_categories()
+        version = Version.objects.get(workspace_id=self.workspace_id).standard_category
+        categories = self.connection.jobs.get_standard_categories(version=version)
         field_names = ['code', 'version', 'description', 'accumulation_name']
         self._sync_data(categories, 'STANDARD_CATEGORY', 'standard_category', self.workspace_id, field_names)
         return []
@@ -125,7 +152,8 @@ def sync_commitments(self):
         """
         Synchronize commitments from Sage Desktop SDK to your application
         """
-        commitments = self.connection.commitments.get_all()
+        version = Version.objects.get(workspace_id=self.workspace_id).commitment
+        commitments = self.connection.commitments.get_all(version=version)
         field_names = [
             'code', 'is_closed', 'version', 'description', 'is_commited',
             'created_on_utc', 'date', 'vendor_id', 'job_id'
@@ -137,12 +165,13 @@ def sync_cost_codes(self):
         """
         Synchronize cost codes from Sage Desktop SDK to your application
         """
-        cost_codes = self.connection.cost_codes.get_all_costcodes()
+        version = Version.objects.get(workspace_id=self.workspace_id).cost_code
+        cost_codes = self.connection.cost_codes.get_all_costcodes(version=version)
         field_names = ['code', 'version', 'job_id']
         self._sync_data(cost_codes, 'COST_CODE', 'cost_code', self.workspace_id, field_names)
         return []
 
-    def sync_categories(self):
+    def sync_cost_categories(self):
         """
          Synchronize categories from Sage Desktop SDK to your application
         """