feat: add error parser (#701)

* feat: add error parser

* chore: add parse error tests (#702)

Co-authored-by: GitHub Actions <[email protected]>

---------

Co-authored-by: GitHub Actions <[email protected]>

.flake8

@@ -28,4 +28,4 @@ max-line-length = 99
 max-complexity = 19
 ban-relative-imports = true
 select = B,C,E,F,N,W,I25
-exclude=*env,*.sql,*.txt,*.sh,.flake8,*.yml,*.yaml
+exclude = *env,*.sql,*.txt,*.sh,.flake8,*.yml,*.yaml,fyle_accounting_mappings/

apps/quickbooks_online/errors/helpers.py

@@ -0,0 +1,90 @@
+import re
+
+from fyle_accounting_mappings.models import DestinationAttribute
+
+field_map = {
+    'Accounts': {'type': 'ACCOUNT', 'article_link': None},
+    'Klasses': {'type': 'CLASS', 'article_link': None},
+    'Names': {'type': 'VENDOR', 'article_link': None},
+    'Depts': {'type': 'DEPARTMENT', 'article_link': None}
+}
+
+
+def error_matcher(error_msg):
+    pattern = r'Invalid Reference Id : (\w+) element id (\d+) not found'
+
+    match = re.search(pattern, error_msg)
+
+    if match:
+        # Extract the QBO element and its ID
+        error_attribute, destination_id = match.groups()
+
+        # Get the field info from the mapping
+        field_info = field_map.get(error_attribute)
+
+        if field_info:
+            # Create a dictionary to store information about the error
+            error_dict = {
+                'attribute_type': field_info['type'],
+                'destination_id': destination_id,
+                'error_attribute': error_attribute,
+                'article_link': field_info['article_link']
+            }
+
+            return error_dict
+
+    # If no match is found, return None
+    return None
+
+
+def get_entity_values(error_dict, workspace_id):
+    '''
+    Get entity values from error dictionary
+    :param error_dict: Error Dictionary containing information about the error
+    :param workspace_id: ID of the workspace
+    :return: Dictionary with 'destination_id' and 'value' if found, otherwise an empty dictionary
+    '''
+    # Fetch the destination attribute based on destination ID and attribute type
+    destination_attribute = DestinationAttribute.objects.filter(
+        destination_id=error_dict['destination_id'],
+        attribute_type=error_dict['attribute_type'].upper(),
+        workspace_id=workspace_id
+    ).first()
+
+    # If the destination attribute is found, return a dictionary with 'destination_id' and 'value'
+    if destination_attribute:
+        return {
+            'destination_id': error_dict['destination_id'],
+            'value': destination_attribute.value,
+            'error_attribute': error_dict['error_attribute'],
+            'attribute_type': error_dict['attribute_type']
+        }
+
+    # If no match is found or destination attribute is not active, return an empty dictionary
+    return {}
+
+
+def replace_destination_id_with_values(input_string, replacement):
+    '''
+    Replace destination ID with corresponding values in the input string
+    :param input_string: Original string containing destination ID placeholders
+    :param replacement: Dictionary with 'destination_id' and 'value' to replace in the string
+    :return: String with destination ID replaced by formatted 'destination_id => value'
+    '''
+
+    # Extract destination ID and value from the replacement dictionary
+    destination_id = replacement['destination_id']
+    value = replacement['value']
+    error_attribute = replacement['error_attribute']
+    attribute_type = replacement['attribute_type']
+
+    # Create a formatted string in the form of 'destination_id => value'
+    arrowed_string_value = f'{destination_id} => {value}'
+    arrowed_string_attribute = f'{error_attribute} => {attribute_type}'
+
+    # Replace occurrences of destination ID in the input string with the formatted string
+    input_string = input_string.replace(destination_id, arrowed_string_value)
+    input_string = input_string.replace(error_attribute, arrowed_string_attribute)
+
+    # Return the modified input string
+    return input_string

apps/quickbooks_online/exceptions.py

@@ -8,6 +8,7 @@
 from apps.fyle.actions import update_failed_expenses
 from apps.fyle.models import ExpenseGroup
 from apps.quickbooks_online.actions import update_last_export_details
+from apps.quickbooks_online.errors.helpers import error_matcher, get_entity_values, replace_destination_id_with_values
 from apps.tasks.models import Error, TaskLog
 from apps.workspaces.models import FyleCredential, QBOCredential
 from fyle_qbo_api.exceptions import BulkError
@@ -35,6 +36,9 @@ def handle_quickbooks_error(exception, expense_group: ExpenseGroup, task_log: Ta
         errors = []
 
         for error in quickbooks_errors:
+            article_link = None
+            attribute_type = None
+            is_parsed = False
             error = {
                 'expense_group_id': expense_group.id,
                 'type': '{0} / {1}'.format(response['Fault']['type'], error['code']),
@@ -44,14 +48,27 @@ def handle_quickbooks_error(exception, expense_group: ExpenseGroup, task_log: Ta
             errors.append(error)
 
             if export_type != 'Bill Payment':
+                error_msg = error['long_description']
+                error_dict = error_matcher(error_msg)
+                if error_dict:
+                    error_entity_values = get_entity_values(error_dict, expense_group.workspace_id)
+                    if error_entity_values:
+                        error_msg = replace_destination_id_with_values(error_msg, error_entity_values)
+                        is_parsed = True
+                        article_link = error_dict['article_link']
+                        attribute_type = error_dict['attribute_type']
+
                 error, created = Error.objects.update_or_create(
                     workspace_id=expense_group.workspace_id,
                     expense_group=expense_group,
                     defaults={
                         'error_title': error['type'],
                         'type': 'QBO_ERROR',
                         'error_detail': error['long_description'],
-                        'is_resolved': False
+                        'is_resolved': False,
+                        'is_parsed': is_parsed,
+                        'attribute_type': attribute_type,
+                        'article_link': article_link
                     })
                 error.increase_repetition_count_by_one(created)
 

apps/tasks/migrations/0012_auto_20241127_0730.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.2.14 on 2024-11-27 07:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('tasks', '0011_error_repetition_count'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='error',
+            name='article_link',
+            field=models.TextField(blank=True, help_text='Article link', null=True),
+        ),
+        migrations.AddField(
+            model_name='error',
+            name='attribute_type',
+            field=models.CharField(blank=True, help_text='Error Attribute type', max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='error',
+            name='is_parsed',
+            field=models.BooleanField(default=False, help_text='Is parsed'),
+        ),
+    ]

apps/tasks/models.py

@@ -1,10 +1,10 @@
 from django.db import models
 from django.db.models import JSONField
-from fyle_accounting_mappings.models import ExpenseAttribute
 
 from apps.fyle.models import ExpenseGroup
 from apps.quickbooks_online.models import Bill, BillPayment, Cheque, CreditCardPurchase, JournalEntry, QBOExpense
 from apps.workspaces.models import Workspace
+from fyle_accounting_mappings.models import ExpenseAttribute
 
 
 def get_default():
@@ -54,6 +54,9 @@ class Error(models.Model):
     is_resolved = models.BooleanField(default=False, help_text='Is resolved')
     error_title = models.CharField(max_length=255, help_text='Error title')
     error_detail = models.TextField(help_text='Error detail')
+    is_parsed = models.BooleanField(default=False, help_text='Is parsed')
+    attribute_type = models.CharField(max_length=255, null=True, blank=True, help_text='Error Attribute type')
+    article_link = models.TextField(null=True, blank=True, help_text='Article link')
     created_at = models.DateTimeField(auto_now_add=True, help_text='Created at datetime')
     updated_at = models.DateTimeField(auto_now=True, help_text='Updated at datetime')
 

scripts/python/parse-errors.py

@@ -0,0 +1,24 @@
+from apps.quickbooks_online.errors.helpers import error_matcher, get_entity_values, replace_destination_id_with_values
+from apps.tasks.models import Error
+
+errors_count = Error.objects.filter(type='QBO_ERROR', is_resolved=False).count()
+print(errors_count)
+page_size = 200
+count = 0
+for offset in range(0, errors_count, page_size):
+    limit = offset + page_size
+    paginated_errors = Error.objects.filter(type='QBO_ERROR', is_resolved=False).order_by('id')[offset:limit]
+    for error in paginated_errors:
+        error_dict = error_matcher(error.error_detail)
+        if error_dict:
+            error_entity_values = get_entity_values(error_dict, error.workspace_id)
+            if error_entity_values:
+                error_msg = replace_destination_id_with_values(error.error_detail, error_entity_values)
+                error.is_parsed = True
+                error.article_link = error_dict['article_link']
+                error.attribute_type = error_dict['attribute_type']
+                error.error_detail = error_msg
+                error.save()
+                count += 1
+
+print(count)

tests/sql_fixtures/migration_fixtures/create_migration.sh

@@ -10,7 +10,7 @@ bash tests/sql_fixtures/reset_db_fixtures/reset_db.sh
 export DATABASE_URL=postgres://postgres:postgres@db:5432/test_qbo_db
 
 # # Running migrations on the fixture database
-# python manage.py migrate
+python manage.py migrate
 
 read -p "Add SQL script paths separated by spaces if any, else press enter to continue? " scripts