From 25548b24951d237a53e464ad35cf50b2bf7690fd Mon Sep 17 00:00:00 2001 From: Shwetabh Kumar Date: Tue, 26 May 2020 13:26:55 +0530 Subject: [PATCH] Quickbooks Integration: Generic Mapping Infra Port --- .github/workflows/pylint.yml | 2 +- .pylintrc | 4 +- Dockerfile | 2 +- apps/fyle/models.py | 112 ++++++-- apps/fyle/tasks.py | 4 +- apps/fyle/urls.py | 8 +- apps/fyle/utils.py | 82 +++++- apps/fyle/views.py | 82 ++++-- apps/mappings/urls.py | 5 +- apps/mappings/utils.py | 149 +--------- apps/quickbooks_online/models.py | 261 ++++++++++++------ apps/quickbooks_online/tasks.py | 39 ++- apps/quickbooks_online/urls.py | 23 +- apps/quickbooks_online/utils.py | 151 ++++++++-- apps/quickbooks_online/views.py | 261 +++++++++++++++--- apps/users/views.py | 1 - apps/workspaces/tasks.py | 7 +- apps/workspaces/utils.py | 11 +- fyle_accounting_mappings/__init__.py | 0 fyle_accounting_mappings/admin.py | 7 + fyle_accounting_mappings/apps.py | 5 + fyle_accounting_mappings/exceptions.py | 21 ++ .../migrations/0001_initial.py | 69 +++++ .../migrations/__init__.py | 0 fyle_accounting_mappings/models.py | 189 +++++++++++++ fyle_accounting_mappings/serializers.py | 44 +++ fyle_accounting_mappings/setup.py | 30 ++ fyle_accounting_mappings/tests.py | 0 fyle_accounting_mappings/urls.py | 23 ++ fyle_accounting_mappings/utils.py | 15 + fyle_accounting_mappings/views.py | 99 +++++++ fyle_qbo_api/settings.py | 5 +- requirements.txt | 1 + scripts/001-mapping_infra.sql | 172 ++++++++++++ setup_template.sh | 8 +- 35 files changed, 1498 insertions(+), 394 deletions(-) create mode 100644 fyle_accounting_mappings/__init__.py create mode 100644 fyle_accounting_mappings/admin.py create mode 100644 fyle_accounting_mappings/apps.py create mode 100644 fyle_accounting_mappings/exceptions.py create mode 100644 fyle_accounting_mappings/migrations/0001_initial.py create mode 100644 fyle_accounting_mappings/migrations/__init__.py create mode 100644 fyle_accounting_mappings/models.py create mode 100644 fyle_accounting_mappings/serializers.py create mode 100644 fyle_accounting_mappings/setup.py create mode 100644 fyle_accounting_mappings/tests.py create mode 100644 fyle_accounting_mappings/urls.py create mode 100644 fyle_accounting_mappings/utils.py create mode 100644 fyle_accounting_mappings/views.py create mode 100644 scripts/001-mapping_infra.sql diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 5f9c1e30..fe71e268 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -14,5 +14,5 @@ jobs: - name: Python Pylin GitHub Action uses: fylein/python-pylint-github-action@v4 with: - args: pip3 install -r requirements.txt && pip install pylint-django && pylint --load-plugins pylint_django --rcfile=.pylintrc **/** + args: pip3 install -r requirements.txt && pip install pylint-django && pylint --load-plugins pylint_django --rcfile=.pylintrc **/**.py diff --git a/.pylintrc b/.pylintrc index 1ce3bd4c..0ee28d26 100644 --- a/.pylintrc +++ b/.pylintrc @@ -151,7 +151,9 @@ disable=print-statement, no-member, simplifiable-if-expression, broad-except, - too-many-arguments + too-many-arguments, + too-many-locals, + too-few-public-methods # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/Dockerfile b/Dockerfile index df1e646f..6a48d089 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ COPY . /fyle-qbo-api/ WORKDIR /fyle-qbo-api # Do linting checks -RUN pylint --load-plugins pylint_django --rcfile=.pylintrc **/** +RUN pylint --load-plugins pylint_django --rcfile=.pylintrc **/**.py # Expose development port EXPOSE 8000 diff --git a/apps/fyle/models.py b/apps/fyle/models.py index 0ace4c79..10aaaddb 100644 --- a/apps/fyle/models.py +++ b/apps/fyle/models.py @@ -6,8 +6,9 @@ from django.db import models from django.contrib.postgres.fields import JSONField +from fyle_accounting_mappings.models import MappingSetting -from apps.workspaces.models import Workspace +from apps.workspaces.models import Workspace, WorkspaceGeneralSettings class Expense(models.Model): @@ -105,38 +106,97 @@ def create_expense_groups_by_report_id_fund_source(expense_objects: List[Expense """ Group expense by report_id and fund_source """ - expense_groups = groupby( - expense_objects, lambda expense: (expense.report_id, expense.employee_email, expense.claim_number, - expense.fund_source) - ) + department_setting: MappingSetting = MappingSetting.objects.filter( + workspace_id=workspace_id, + destination_field='DEPARTMENT' + ).first() - expense_group_objects = [] + general_settings = WorkspaceGeneralSettings.objects.get(workspace_id=workspace_id) + + reimbursable_expenses = list(filter(lambda expense: expense.fund_source == 'PERSONAL', expense_objects)) - for expense_group, _ in expense_groups: - report_id = expense_group[0] - employee_email = expense_group[1] - claim_number = expense_group[2] - fund_source = expense_group[3] + ccc_expenses = list(filter(lambda expense: expense.fund_source == 'CCC', expense_objects)) - expense_ids = Expense.objects.filter(report_id=report_id, fund_source=fund_source).values_list( - 'id', flat=True + if department_setting and general_settings.reimbursable_expenses_object != 'JOURNAL_ENTRY': + reimbursable_expense_groups = groupby( + reimbursable_expenses, lambda expense: ( + expense.report_id, expense.employee_email, + expense.claim_number, expense.fund_source, + expense.project if department_setting.source_field == 'PROJECT' else expense.cost_center + ) + ) + else: + reimbursable_expense_groups = groupby( + reimbursable_expenses, lambda expense: ( + expense.report_id, expense.employee_email, + expense.claim_number, expense.fund_source + ) ) - expense_group_object, _ = ExpenseGroup.objects.update_or_create( - fyle_group_id=report_id + '-' + fund_source.lower(), - workspace_id=workspace_id, - fund_source=fund_source, - defaults={ - 'description': { - 'employee_email': employee_email, - 'claim_number': claim_number, - 'fund_source': fund_source + group_types = [reimbursable_expense_groups] + + if general_settings.corporate_credit_card_expenses_object and ccc_expenses: + if department_setting and general_settings.corporate_credit_card_expenses_object != 'JOURNAL_ENTRY': + ccc_expense_groups = groupby( + ccc_expenses, lambda expense: ( + expense.report_id, expense.employee_email, + expense.claim_number, expense.fund_source, + expense.project if department_setting.source_field == 'PROJECT' else expense.cost_center + ) + ) + else: + ccc_expense_groups = groupby( + ccc_expenses, lambda expense: ( + expense.report_id, expense.employee_email, + expense.claim_number, expense.fund_source + ) + ) + group_types.append(ccc_expense_groups) + + expense_group_objects = [] + + for expense_groups in group_types: + for expense_group, _ in expense_groups: + report_id = expense_group[0] + employee_email = expense_group[1] + claim_number = expense_group[2] + fund_source = expense_group[3] + department = None + if len(expense_group) > 4: + department = expense_group[4] + + kwargs = {} + + if department: + kwargs = { + '{0}'.format(department_setting.source_field.lower()): department, } - } - ) - expense_group_object.expenses.add(*expense_ids) + expense_ids = Expense.objects.filter( + report_id=report_id, + fund_source=fund_source, + **kwargs + ).values_list( + 'id', flat=True + ) + + expense_group_object, _ = ExpenseGroup.objects.update_or_create( + fyle_group_id='{0}-{1}'.format(claim_number, fund_source) if not department + else '{0}-{1}-{2}'.format(claim_number, fund_source, department), + workspace_id=workspace_id, + fund_source=fund_source, + defaults={ + 'description': { + 'employee_email': employee_email, + 'claim_number': claim_number, + 'fund_source': fund_source, + department_setting.source_field.lower(): department + } + } + ) + + expense_group_object.expenses.add(*expense_ids) - expense_group_objects.append(expense_group_object) + expense_group_objects.append(expense_group_object) return expense_group_objects diff --git a/apps/fyle/tasks.py b/apps/fyle/tasks.py index b8940bdb..66021afc 100644 --- a/apps/fyle/tasks.py +++ b/apps/fyle/tasks.py @@ -26,7 +26,7 @@ def schedule_expense_group_creation(workspace_id: int, user: str): """ fyle_credentials = FyleCredential.objects.get( workspace_id=workspace_id) - fyle_connector = FyleConnector(fyle_credentials.refresh_token) + fyle_connector = FyleConnector(fyle_credentials.refresh_token, workspace_id) fyle_sdk_connection = fyle_connector.connection jobs = FyleJobsSDK(settings.FYLE_JOBS_URL, fyle_sdk_connection) @@ -93,7 +93,7 @@ def async_create_expense_groups(workspace_id: int, state: List[str], fund_source fyle_credentials = FyleCredential.objects.get(workspace_id=workspace_id) - fyle_connector = FyleConnector(fyle_credentials.refresh_token) + fyle_connector = FyleConnector(fyle_credentials.refresh_token, workspace_id) expenses = fyle_connector.get_expenses( state=state, diff --git a/apps/fyle/urls.py b/apps/fyle/urls.py index 2f922e32..7e80632a 100644 --- a/apps/fyle/urls.py +++ b/apps/fyle/urls.py @@ -24,8 +24,8 @@ path('expense_groups/trigger/', ExpenseGroupScheduleView.as_view()), path('expense_groups//', ExpenseGroupByIdView.as_view()), path('expense_groups//expenses/', ExpenseView.as_view()), - path('employees/', EmployeeView.as_view({'get': 'get_employees'})), - path('categories/', CategoryView.as_view({'get': 'get_categories'})), - path('cost_centers/', CostCenterView.as_view({'get': 'get_cost_centers'})), - path('projects/', ProjectView.as_view({'get': 'get_projects'})) + path('employees/', EmployeeView.as_view()), + path('categories/', CategoryView.as_view()), + path('cost_centers/', CostCenterView.as_view()), + path('projects/', ProjectView.as_view()) ] diff --git a/apps/fyle/utils.py b/apps/fyle/utils.py index 8359c6e1..a0973cfc 100644 --- a/apps/fyle/utils.py +++ b/apps/fyle/utils.py @@ -4,15 +4,19 @@ from fylesdk import FyleSDK +from fyle_accounting_mappings.models import ExpenseAttribute + class FyleConnector: """ Fyle utility functions """ - def __init__(self, refresh_token): + + def __init__(self, refresh_token, workspace_id=None): client_id = settings.FYLE_CLIENT_ID client_secret = settings.FYLE_CLIENT_SECRET base_url = settings.FYLE_BASE_URL + self.workspace_id = workspace_id self.connection = FyleSDK( base_url=base_url, @@ -39,26 +43,86 @@ def get_expenses(self, state: List[str], updated_at: List[str], fund_source: Lis expenses)) return expenses - def get_employees(self): + def sync_employees(self): """ Get employees from fyle """ - return self.connection.Employees.get_all() + employees = self.connection.Employees.get_all() + + employee_attributes = [] + + for employee in employees: + employee_attributes.append({ + 'attribute_type': 'EMPLOYEE', + 'display_name': 'Employee', + 'value': employee['employee_email'], + 'source_id': employee['id'] + }) + + employee_attributes = ExpenseAttribute.bulk_upsert_expense_attributes(employee_attributes, self.workspace_id) - def get_categories(self, active_only: bool): + return employee_attributes + + def sync_categories(self, active_only: bool): """ Get categories from fyle """ - return self.connection.Categories.get(active_only=active_only)['data'] + categories = self.connection.Categories.get(active_only=active_only)['data'] + + category_attributes = [] + + for category in categories: + if category['name'] != category['sub_category']: + category['name'] = '{0} / {1}'.format(category['name'], category['sub_category']) + + category_attributes.append({ + 'attribute_type': 'CATEGORY', + 'display_name': 'Category', + 'value': category['name'], + 'source_id': category['id'] + }) + + category_attributes = ExpenseAttribute.bulk_upsert_expense_attributes(category_attributes, self.workspace_id) - def get_cost_centers(self, active_only: bool): + return category_attributes + + def sync_cost_centers(self, active_only: bool): """ Get cost centers from fyle """ - return self.connection.CostCenters.get(active_only=active_only)['data'] + cost_centers = self.connection.CostCenters.get(active_only=active_only)['data'] + + cost_center_attributes = [] + + for cost_center in cost_centers: + cost_center_attributes.append({ + 'attribute_type': 'COST_CENTER', + 'display_name': 'Cost Center', + 'value': cost_center['name'], + 'source_id': cost_center['id'] + }) + + cost_center_attributes = ExpenseAttribute.bulk_upsert_expense_attributes( + cost_center_attributes, self.workspace_id) + + return cost_center_attributes - def get_projects(self, active_only: bool): + def sync_projects(self, active_only: bool): """ Get projects from fyle """ - return self.connection.Projects.get(active_only=active_only)['data'] + projects = self.connection.Projects.get(active_only=active_only)['data'] + + project_attributes = [] + + for project in projects: + project_attributes.append({ + 'attribute_type': 'PROJECT', + 'display_name': 'Project', + 'value': project['name'], + 'source_id': project['id'] + }) + + project_attributes = ExpenseAttribute.bulk_upsert_expense_attributes(project_attributes, self.workspace_id) + + return project_attributes diff --git a/apps/fyle/views.py b/apps/fyle/views.py index 8eccfef5..7d9ecd6f 100644 --- a/apps/fyle/views.py +++ b/apps/fyle/views.py @@ -1,10 +1,12 @@ from rest_framework.views import status from rest_framework import generics -from rest_framework import viewsets from rest_framework.response import Response -from apps.tasks.models import TaskLog +from fyle_accounting_mappings.models import ExpenseAttribute +from fyle_accounting_mappings.serializers import ExpenseAttributeSerializer + from apps.workspaces.models import FyleCredential, WorkspaceGeneralSettings +from apps.tasks.models import TaskLog from .tasks import create_expense_groups, schedule_expense_group_creation from .utils import FyleConnector @@ -158,12 +160,19 @@ def get(self, request, *args, **kwargs): ) -class EmployeeView(viewsets.ViewSet): +class EmployeeView(generics.ListCreateAPIView): """ Employee view """ - def get_employees(self, request, **kwargs): + serializer_class = ExpenseAttributeSerializer + pagination_class = None + + def get_queryset(self): + return ExpenseAttribute.objects.filter( + attribute_type='EMPLOYEE', workspace_id=self.kwargs['workspace_id']).order_by('value') + + def post(self, request, *args, **kwargs): """ Get employees from Fyle """ @@ -171,12 +180,12 @@ def get_employees(self, request, **kwargs): fyle_credentials = FyleCredential.objects.get( workspace_id=kwargs['workspace_id']) - fyle_connector = FyleConnector(fyle_credentials.refresh_token) + fyle_connector = FyleConnector(fyle_credentials.refresh_token, kwargs['workspace_id']) - employees = fyle_connector.get_employees() + employee_attributes = fyle_connector.sync_employees() return Response( - data=employees, + data=self.serializer_class(employee_attributes, many=True).data, status=status.HTTP_200_OK ) except FyleCredential.DoesNotExist: @@ -188,12 +197,19 @@ def get_employees(self, request, **kwargs): ) -class CategoryView(viewsets.ViewSet): +class CategoryView(generics.ListCreateAPIView): """ Category view """ - def get_categories(self, request, **kwargs): + serializer_class = ExpenseAttributeSerializer + pagination_class = None + + def get_queryset(self): + return ExpenseAttribute.objects.filter( + attribute_type='CATEGORY', workspace_id=self.kwargs['workspace_id']).order_by('value') + + def post(self, request, *args, **kwargs): """ Get categories from Fyle """ @@ -202,12 +218,12 @@ def get_categories(self, request, **kwargs): fyle_credentials = FyleCredential.objects.get( workspace_id=kwargs['workspace_id']) - fyle_connector = FyleConnector(fyle_credentials.refresh_token) + fyle_connector = FyleConnector(fyle_credentials.refresh_token, kwargs['workspace_id']) - categories = fyle_connector.get_categories(active_only=active_only) + category_attributes = fyle_connector.sync_categories(active_only=active_only) return Response( - data=categories, + data=self.serializer_class(category_attributes, many=True).data, status=status.HTTP_200_OK ) except FyleCredential.DoesNotExist: @@ -219,27 +235,33 @@ def get_categories(self, request, **kwargs): ) -class CostCenterView(viewsets.ViewSet): +class CostCenterView(generics.ListCreateAPIView): """ - Cost center view + Category view """ - def get_cost_centers(self, request, **kwargs): + serializer_class = ExpenseAttributeSerializer + pagination_class = None + + def get_queryset(self): + return ExpenseAttribute.objects.filter( + attribute_type='COST_CENTER', workspace_id=self.kwargs['workspace_id']).order_by('value') + + def post(self, request, *args, **kwargs): """ - Get cost centers from Fyle + Get categories from Fyle """ try: active_only = request.GET.get('active_only', False) fyle_credentials = FyleCredential.objects.get( workspace_id=kwargs['workspace_id']) - fyle_connector = FyleConnector(fyle_credentials.refresh_token) + fyle_connector = FyleConnector(fyle_credentials.refresh_token, kwargs['workspace_id']) - cost_centers = fyle_connector.get_cost_centers( - active_only=active_only) + cost_center_attributes = fyle_connector.sync_cost_centers(active_only=active_only) return Response( - data=cost_centers, + data=self.serializer_class(cost_center_attributes, many=True).data, status=status.HTTP_200_OK ) except FyleCredential.DoesNotExist: @@ -251,26 +273,32 @@ def get_cost_centers(self, request, **kwargs): ) -class ProjectView(viewsets.ViewSet): +class ProjectView(generics.ListCreateAPIView): """ Project view """ + serializer_class = ExpenseAttributeSerializer + pagination_class = None + + def get_queryset(self): + return ExpenseAttribute.objects.filter( + attribute_type='PROJECT', workspace_id=self.kwargs['workspace_id']).order_by('value') - def get_projects(self, request, **kwargs): + def post(self, request, *args, **kwargs): """ - Get projects from Fyle + Get categories from Fyle """ try: active_only = request.GET.get('active_only', False) fyle_credentials = FyleCredential.objects.get( workspace_id=kwargs['workspace_id']) - fyle_connector = FyleConnector(fyle_credentials.refresh_token) + fyle_connector = FyleConnector(fyle_credentials.refresh_token, kwargs['workspace_id']) - projects = fyle_connector.get_projects(active_only=active_only) + project_attributes = fyle_connector.sync_projects(active_only=active_only) return Response( - data=projects, + data=self.serializer_class(project_attributes, many=True).data, status=status.HTTP_200_OK ) except FyleCredential.DoesNotExist: @@ -289,7 +317,7 @@ def get(self, request, *args, **kwargs): fyle_credentials = FyleCredential.objects.get( workspace_id=kwargs.get('workspace_id')) - fyle_connector = FyleConnector(fyle_credentials.refresh_token) + fyle_connector = FyleConnector(fyle_credentials.refresh_token, kwargs['workspace_id']) employee_profile = fyle_connector.get_employee_profile() diff --git a/apps/mappings/urls.py b/apps/mappings/urls.py index 4d79ab30..d8e165bf 100644 --- a/apps/mappings/urls.py +++ b/apps/mappings/urls.py @@ -13,7 +13,7 @@ 1. Import the include() function: from django.urls import include, path 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ -from django.urls import path +from django.urls import path, include from .views import GeneralMappingView, EmployeeMappingView, CategoryMappingView, \ CostCenterMappingView, ProjectMappingView @@ -23,5 +23,6 @@ path('employees/', EmployeeMappingView.as_view()), path('categories/', CategoryMappingView.as_view()), path('cost_centers/', CostCenterMappingView.as_view()), - path('projects/', ProjectMappingView.as_view()) + path('projects/', ProjectMappingView.as_view()), + path('', include('fyle_accounting_mappings.urls')) ] diff --git a/apps/mappings/utils.py b/apps/mappings/utils.py index 137b2bc3..6eca0d28 100644 --- a/apps/mappings/utils.py +++ b/apps/mappings/utils.py @@ -1,9 +1,13 @@ from typing import Dict +from django.db.models import Q + +from fyle_accounting_mappings.models import MappingSetting + from apps.workspaces.models import WorkspaceGeneralSettings from fyle_qbo_api.utils import assert_valid -from .models import GeneralMapping, EmployeeMapping, CategoryMapping, CostCenterMapping, ProjectMapping +from .models import GeneralMapping class MappingUtils: @@ -16,8 +20,7 @@ def create_or_update_general_mapping(self, general_mapping: Dict): :param general_mapping: general mapping payload :return: """ - queryset = WorkspaceGeneralSettings.objects.all() - general_settings = queryset.get(workspace_id=self.__workspace_id) + general_settings = WorkspaceGeneralSettings.objects.get(workspace_id=self.__workspace_id) params = { 'accounts_payable_name': None, @@ -28,7 +31,12 @@ def create_or_update_general_mapping(self, general_mapping: Dict): 'default_ccc_account_id': None } - if general_settings.employee_field_mapping == 'VENDOR': + mapping_setting = MappingSetting.objects.filter( + Q(destination_field='VENDOR') | Q(destination_field='EMPLOYEE'), + source_field='EMPLOYEE', workspace_id=self.__workspace_id + ).first() + + if mapping_setting.destination_field == 'VENDOR': assert_valid('accounts_payable_name' in general_mapping and general_mapping['accounts_payable_name'], 'account payable account name field is blank') assert_valid('accounts_payable_id' in general_mapping and general_mapping['accounts_payable_id'], @@ -37,7 +45,7 @@ def create_or_update_general_mapping(self, general_mapping: Dict): params['accounts_payable_name'] = general_mapping.get('accounts_payable_name') params['accounts_payable_id'] = general_mapping.get('accounts_payable_id') - if general_settings.employee_field_mapping == 'EMPLOYEE': + if mapping_setting.destination_field == 'EMPLOYEE': assert_valid('bank_account_name' in general_mapping and general_mapping['bank_account_name'], 'bank account name field is blank') assert_valid('bank_account_id' in general_mapping and general_mapping['bank_account_id'], @@ -60,134 +68,3 @@ def create_or_update_general_mapping(self, general_mapping: Dict): defaults=params ) return general_mapping - - def create_or_update_employee_mapping(self, employee_mapping: Dict): - """ - Create or update employee mappings - :param employee_mapping: employee mapping payload - :return: employee mappings objects - """ - params = { - 'vendor_display_name': None, - 'vendor_id': None, - 'employee_display_name': None, - 'employee_id': None, - 'ccc_account_name': None, - 'ccc_account_id': None - } - - general_settings_queryset = WorkspaceGeneralSettings.objects.all() - general_settings = general_settings_queryset.get(workspace_id=self.__workspace_id) - - assert_valid('employee_email' in employee_mapping and employee_mapping['employee_email'], - 'employee email field is blank') - - if general_settings.employee_field_mapping == 'VENDOR': - assert_valid('vendor_id' in employee_mapping and employee_mapping['vendor_id'], - 'vendor id field is blank') - assert_valid('vendor_display_name' in employee_mapping and employee_mapping['vendor_display_name'], - 'vendor display name is missing') - - params['vendor_display_name'] = employee_mapping.get('vendor_display_name') - params['vendor_id'] = employee_mapping.get('vendor_id') - - elif general_settings.employee_field_mapping == 'EMPLOYEE': - assert_valid('employee_display_name' in employee_mapping and employee_mapping['employee_display_name'], - 'employee_display_name field is blank') - assert_valid('employee_id' in employee_mapping and employee_mapping['employee_id'], - 'employee_id field is blank') - - params['employee_display_name'] = employee_mapping.get('employee_display_name') - params['employee_id'] = employee_mapping.get('employee_id') - - if general_settings.corporate_credit_card_expenses_object: - assert_valid('ccc_account_name' in employee_mapping and employee_mapping['ccc_account_name'], - 'ccc account name field is blank') - assert_valid('ccc_account_id' in employee_mapping and employee_mapping['ccc_account_id'], - 'ccc account id field is blank') - - params['ccc_account_name'] = employee_mapping.get('ccc_account_name') - params['ccc_account_id'] = employee_mapping.get('ccc_account_id') - - employee_mapping_object, _ = EmployeeMapping.objects.update_or_create( - employee_email=employee_mapping['employee_email'].lower(), - workspace_id=self.__workspace_id, - defaults=params - ) - - return employee_mapping_object - - def create_or_update_category_mapping(self, category_mapping: Dict): - """ - Create or update category mappings - :param category_mapping: category mapping payload - :return: category mappings objects - """ - assert_valid('category' in category_mapping and category_mapping['category'], - 'category field is blank') - assert_valid('sub_category' in category_mapping and category_mapping['sub_category'], - 'sub_category field is blank') - assert_valid('account_name' in category_mapping and category_mapping['account_name'], - 'account name field is blank') - assert_valid('account_id' in category_mapping and category_mapping['account_id'], - 'account id field is blank') - - category_mapping_object, _ = CategoryMapping.objects.update_or_create( - category=category_mapping['category'], - sub_category=category_mapping['sub_category'], - workspace_id=self.__workspace_id, - defaults={ - 'account_name': category_mapping['account_name'], - 'account_id': category_mapping['account_id'] - } - ) - - return category_mapping_object - - def create_or_update_cost_center_mapping(self, cost_center_mapping: Dict): - """ - Create or update cost_center mappings - :param cost_center_mapping: cost_center mapping payload - :return: cost_center mappings objects - """ - assert_valid('cost_center' in cost_center_mapping and cost_center_mapping['cost_center'], - 'cost_center field is blank') - assert_valid('class_name' in cost_center_mapping and cost_center_mapping['class_name'], - 'class name field is blank') - assert_valid('class_id' in cost_center_mapping and cost_center_mapping['class_id'], - 'class id field is blank') - - cost_center_mapping_object, _ = CostCenterMapping.objects.update_or_create( - cost_center=cost_center_mapping['cost_center'], - workspace_id=self.__workspace_id, - defaults={ - 'class_name': cost_center_mapping['class_name'], - 'class_id': cost_center_mapping['class_id'] - } - ) - - return cost_center_mapping_object - - def create_or_update_project_mapping(self, project_mapping: Dict): - """ - Create or update project mappings - :param project_mapping: project mapping payload - :return: project mappings objects - """ - assert_valid('project' in project_mapping and project_mapping['project'], - 'project field is blank') - assert_valid('customer_display_name' in project_mapping and project_mapping['customer_display_name'], - 'customer name field is blank') - assert_valid('customer_id' in project_mapping and project_mapping['customer_id'], - 'customer id field is blank') - - project_mapping_object, _ = ProjectMapping.objects.update_or_create( - project=project_mapping['project'], - workspace_id=self.__workspace_id, - defaults={ - 'customer_display_name': project_mapping['customer_display_name'], - 'customer_id': project_mapping['customer_id'] - } - ) - - return project_mapping_object diff --git a/apps/quickbooks_online/models.py b/apps/quickbooks_online/models.py index 2e76e2d6..3608111e 100644 --- a/apps/quickbooks_online/models.py +++ b/apps/quickbooks_online/models.py @@ -4,10 +4,99 @@ from datetime import datetime from django.db import models +from django.db.models import Q + +from fyle_accounting_mappings.models import Mapping, MappingSetting from apps.fyle.models import ExpenseGroup, Expense -from apps.mappings.models import GeneralMapping, EmployeeMapping, CategoryMapping, CostCenterMapping, ProjectMapping -from apps.workspaces.models import WorkspaceGeneralSettings +from apps.mappings.models import GeneralMapping + + +def get_class_id_or_none(expense_group: ExpenseGroup, lineitem: Expense): + class_setting: MappingSetting = MappingSetting.objects.filter( + workspace_id=expense_group.workspace_id, + destination_field='CLASS' + ).first() + + class_id = None + + if class_setting: + source_value = None + + if class_setting.source_field == 'PROJECT': + source_value = lineitem.project + elif class_setting.source_field == 'COST_CENTER': + source_value = lineitem.cost_center + + mapping: Mapping = Mapping.objects.filter( + source_type=class_setting.source_field, + destination_type='CLASS', + source__value=source_value, + workspace_id=expense_group.workspace_id + ).first() + + if mapping: + class_id = mapping.destination.destination_id + return class_id + + +def get_customer_id_or_none(expense_group: ExpenseGroup, lineitem: Expense): + customer_setting: MappingSetting = MappingSetting.objects.filter( + workspace_id=expense_group.workspace_id, + destination_field='CUSTOMER' + ).first() + + customer_id = None + + if customer_setting: + source_value = None + + if customer_setting.source_field == 'PROJECT': + source_value = lineitem.project + elif customer_setting.source_field == 'COST_CENTER': + source_value = lineitem.cost_center + + mapping: Mapping = Mapping.objects.filter( + source_type=customer_setting.source_field, + destination_type='CUSTOMER', + source__value=source_value, + workspace_id=expense_group.workspace_id + ).first() + + if mapping: + customer_id = mapping.destination.destination_id + return customer_id + + +def get_department_id_or_none(expense_group: ExpenseGroup, lineitem: Expense = None): + department_setting: MappingSetting = MappingSetting.objects.filter( + workspace_id=expense_group.workspace_id, + destination_field='DEPARTMENT' + ).first() + + department_id = None + + if department_setting: + source_value = None + + if lineitem: + if department_setting.source_field == 'PROJECT': + source_value = lineitem.project + elif department_setting.source_field == 'COST_CENTER': + source_value = lineitem.cost_center + else: + source_value = expense_group.description[department_setting.source_field.lower()] + + mapping: Mapping = Mapping.objects.filter( + source_type=department_setting.source_field, + destination_type='DEPARTMENT', + source__value=source_value, + workspace_id=expense_group.workspace_id + ).first() + + if mapping: + department_id = mapping.destination.destination_id + return department_id class Bill(models.Model): @@ -37,16 +126,20 @@ def create_bill(expense_group: ExpenseGroup): expense = expense_group.expenses.first() + department_id = get_department_id_or_none(expense_group) + general_mappings = GeneralMapping.objects.get(workspace_id=expense_group.workspace_id) bill_object, _ = Bill.objects.update_or_create( expense_group=expense_group, defaults={ 'accounts_payable_id': general_mappings.accounts_payable_id, - 'vendor_id': EmployeeMapping.objects.get( - employee_email=description.get('employee_email'), + 'vendor_id': Mapping.objects.get( + source_type='EMPLOYEE', + destination_type='VENDOR', + source__value=description.get('employee_email'), workspace_id=expense_group.workspace_id - ).vendor_id, - 'department_id': None, + ).destination.destination_id, + 'department_id': department_id, 'transaction_date': datetime.now().strftime("%Y-%m-%d"), 'private_note': 'Report {0} / {1} exported on {2}'.format( expense.claim_number, expense.report_id, datetime.now().strftime("%Y-%m-%d") @@ -85,25 +178,25 @@ def create_bill_lineitems(expense_group: ExpenseGroup): bill_lineitem_objects = [] for lineitem in expenses: - account = CategoryMapping.objects.filter(category=lineitem.category, - sub_category=lineitem.sub_category, - workspace_id=expense_group.workspace_id).first() - - cost_center_mapping = CostCenterMapping.objects.filter( - cost_center=lineitem.cost_center, workspace_id=expense_group.workspace_id).first() + category = lineitem.category if lineitem.category == lineitem.sub_category else '{0} / {1}'.format( + lineitem.category, lineitem.sub_category) - project_mapping = ProjectMapping.objects.filter( - project=lineitem.project, workspace_id=expense_group.workspace_id).first() + account: Mapping = Mapping.objects.filter( + source_type='CATEGORY', + destination_type='ACCOUNT', + source__value=category, + workspace_id=expense_group.workspace_id + ).first() - class_id = cost_center_mapping.class_id if cost_center_mapping else None + class_id = get_class_id_or_none(expense_group, lineitem) - customer_id = project_mapping.customer_id if project_mapping else None + customer_id = get_customer_id_or_none(expense_group, lineitem) bill_lineitem_object, _ = BillLineitem.objects.update_or_create( bill=bill, expense_id=lineitem.id, defaults={ - 'account_id': account.account_id if account else None, + 'account_id': account.destination.destination_id if account else None, 'class_id': class_id, 'customer_id': customer_id, 'amount': lineitem.amount, @@ -144,15 +237,20 @@ def create_cheque(expense_group: ExpenseGroup): expense = expense_group.expenses.first() general_mappings = GeneralMapping.objects.get(workspace_id=expense_group.workspace_id) + + department_id = get_department_id_or_none(expense_group) + cheque_object, _ = Cheque.objects.update_or_create( expense_group=expense_group, defaults={ 'bank_account_id': general_mappings.bank_account_id, - 'entity_id': EmployeeMapping.objects.get( - employee_email=description.get('employee_email'), + 'entity_id': Mapping.objects.get( + source_type='EMPLOYEE', + destination_type='EMPLOYEE', + source__value=description.get('employee_email'), workspace_id=expense_group.workspace_id - ).employee_id, - 'department_id': None, + ).destination.destination_id, + 'department_id': department_id, 'transaction_date': datetime.now().strftime("%Y-%m-%d"), 'private_note': 'Report {0} / {1} exported on {2}'.format( expense.claim_number, expense.report_id, datetime.now().strftime("%Y-%m-%d") @@ -191,24 +289,25 @@ def create_cheque_lineitems(expense_group: ExpenseGroup): cheque_lineitem_objects = [] for lineitem in expenses: - account = CategoryMapping.objects.filter(category=lineitem.category, - sub_category=lineitem.sub_category).first() - - cost_center_mapping = CostCenterMapping.objects.filter( - cost_center=lineitem.cost_center, workspace_id=expense_group.workspace_id).first() + category = lineitem.category if lineitem.category == lineitem.sub_category else '{0} / {1}'.format( + lineitem.category, lineitem.sub_category) - project_mapping = ProjectMapping.objects.filter( - project=lineitem.project, workspace_id=expense_group.workspace_id).first() + account: Mapping = Mapping.objects.filter( + source_type='CATEGORY', + destination_type='ACCOUNT', + source__value=category, + workspace_id=expense_group.workspace_id + ).first() - class_id = cost_center_mapping.class_id if cost_center_mapping else None + class_id = get_class_id_or_none(expense_group, lineitem) - customer_id = project_mapping.customer_id if project_mapping else None + customer_id = get_customer_id_or_none(expense_group, lineitem) cheque_lineitem_object, _ = ChequeLineitem.objects.update_or_create( cheque=cheque, expense_id=lineitem.id, defaults={ - 'account_id': account.account_id if account else None, + 'account_id': account.destination.destination_id if account else None, 'class_id': class_id, 'customer_id': customer_id, 'amount': lineitem.amount, @@ -247,21 +346,24 @@ def create_credit_card_purchase(expense_group: ExpenseGroup): description = expense_group.description expense = expense_group.expenses.first() - general_settings_queryset = WorkspaceGeneralSettings.objects.all() - general_settings = general_settings_queryset.get(workspace_id=expense_group.workspace_id) + department_id = get_department_id_or_none(expense_group) credit_card_purchase_object, _ = CreditCardPurchase.objects.update_or_create( expense_group=expense_group, defaults={ - 'ccc_account_id': EmployeeMapping.objects.get( - employee_email=description.get('employee_email'), + 'ccc_account_id': Mapping.objects.get( + destination_type='CREDIT_CARD_ACCOUNT', + source_type='EMPLOYEE', + source__value=description.get('employee_email'), + workspace_id=expense_group.workspace_id + ).destination.destination_id, + 'department_id': department_id, + 'entity_id': Mapping.objects.get( + Q(destination_type='EMPLOYEE') | Q(destination_type='VENDOR'), + source_type='EMPLOYEE', + source__value=description.get('employee_email'), workspace_id=expense_group.workspace_id - ).ccc_account_id, - 'entity_id': EmployeeMapping.objects.get(employee_email=description.get('employee_email'), - workspace_id=expense_group.workspace_id).employee_id - if general_settings.employee_field_mapping == 'EMPLOYEE' else - EmployeeMapping.objects.get(employee_email=description.get('employee_email'), - workspace_id=expense_group.workspace_id).vendor_id, + ).destination.destination_id, 'transaction_date': datetime.now().strftime("%Y-%m-%d"), 'private_note': 'Report {0} / {1} exported on {2}'.format( expense.claim_number, expense.report_id, datetime.now().strftime("%Y-%m-%d") @@ -301,24 +403,25 @@ def create_credit_card_purchase_lineitems(expense_group: ExpenseGroup): credit_card_purchase_lineitem_objects = [] for lineitem in expenses: - account = CategoryMapping.objects.filter(category=lineitem.category, - sub_category=lineitem.sub_category).first() - - cost_center_mapping = CostCenterMapping.objects.filter( - cost_center=lineitem.cost_center, workspace_id=expense_group.workspace_id).first() + category = lineitem.category if lineitem.category == lineitem.sub_category else '{0} / {1}'.format( + lineitem.category, lineitem.sub_category) - project_mapping = ProjectMapping.objects.filter( - project=lineitem.project, workspace_id=expense_group.workspace_id).first() + account: Mapping = Mapping.objects.filter( + source_type='CATEGORY', + destination_type='ACCOUNT', + source__value=category, + workspace_id=expense_group.workspace_id + ).first() - class_id = cost_center_mapping.class_id if cost_center_mapping else None + class_id = get_class_id_or_none(expense_group, lineitem) - customer_id = project_mapping.customer_id if project_mapping else None + customer_id = get_customer_id_or_none(expense_group, lineitem) credit_card_purchase_lineitem_object, _ = CreditCardPurchaseLineitem.objects.update_or_create( credit_card_purchase=credit_card_purchase, expense_id=lineitem.id, defaults={ - 'account_id': account.account_id if account else None, + 'account_id': account.destination.destination_id if account else None, 'class_id': class_id, 'customer_id': customer_id, 'amount': lineitem.amount, @@ -398,63 +501,67 @@ def create_journal_entry_lineitems(expense_group: ExpenseGroup): description = expense_group.description - general_settings_queryset = WorkspaceGeneralSettings.objects.all() - general_settings = general_settings_queryset.get(workspace_id=expense_group.workspace_id) - debit_account_id = None entity_type = None + entity = Mapping.objects.get( + Q(destination_type='EMPLOYEE') | Q(destination_type='VENDOR'), + source_type='EMPLOYEE', + source__value=description.get('employee_email'), + workspace_id=expense_group.workspace_id + ) + if expense_group.fund_source == 'PERSONAL': - if general_settings.employee_field_mapping == 'VENDOR': + if entity.destination_type == 'VENDOR': debit_account_id = GeneralMapping.objects.get( workspace_id=expense_group.workspace_id).accounts_payable_id - elif general_settings.employee_field_mapping == 'EMPLOYEE': + elif entity.destination_type == 'EMPLOYEE': debit_account_id = GeneralMapping.objects.get( workspace_id=expense_group.workspace_id).bank_account_id elif expense_group.fund_source == 'CCC': - debit_account_id = EmployeeMapping.objects.get( - employee_email=description.get('employee_email'), + debit_account_id = Mapping.objects.get( + source_type='EMPLOYEE', + destination_type='CREDIT_CARD_ACCOUNT', + source__value=description.get('employee_email'), workspace_id=expense_group.workspace_id - ).ccc_account_id + ).destination.destination_id - if general_settings.employee_field_mapping == 'EMPLOYEE': + if entity.destination_type == 'EMPLOYEE': entity_type = 'Employee' - elif general_settings.employee_field_mapping == 'VENDOR': + elif entity.destination_type == 'VENDOR': entity_type = 'Vendor' journal_entry_lineitem_objects = [] for lineitem in expenses: - account = CategoryMapping.objects.filter(category=lineitem.category, - sub_category=lineitem.sub_category).first() + category = lineitem.category if lineitem.category == lineitem.sub_category else '{0} / {1}'.format( + lineitem.category, lineitem.sub_category) - cost_center_mapping = CostCenterMapping.objects.filter( - cost_center=lineitem.cost_center, workspace_id=expense_group.workspace_id).first() - - project_mapping = ProjectMapping.objects.filter( - project=lineitem.project, workspace_id=expense_group.workspace_id).first() + account: Mapping = Mapping.objects.filter( + source_type='CATEGORY', + destination_type='ACCOUNT', + source__value=category, + workspace_id=expense_group.workspace_id + ).first() - entity_id = EmployeeMapping.objects.get(employee_email=description.get('employee_email'), - workspace_id=expense_group.workspace_id).employee_id \ - if general_settings.employee_field_mapping == 'EMPLOYEE' \ - else EmployeeMapping.objects.get(employee_email=description.get('employee_email'), - workspace_id=expense_group.workspace_id).vendor_id + class_id = get_class_id_or_none(expense_group, lineitem) - class_id = cost_center_mapping.class_id if cost_center_mapping else None + customer_id = get_customer_id_or_none(expense_group, lineitem) - customer_id = project_mapping.customer_id if project_mapping else None + department_id = get_department_id_or_none(expense_group) journal_entry_lineitem_object, _ = JournalEntryLineitem.objects.update_or_create( journal_entry=qbo_journal_entry, expense_id=lineitem.id, defaults={ 'debit_account_id': debit_account_id, - 'account_id': account.account_id if account else None, + 'account_id': account.destination.destination_id if account else None, 'class_id': class_id, - 'entity_id': entity_id, + 'entity_id': entity.destination.destination_id, 'entity_type': entity_type, 'customer_id': customer_id, 'amount': lineitem.amount, + 'department_id': department_id, 'description': lineitem.purpose } ) diff --git a/apps/quickbooks_online/tasks.py b/apps/quickbooks_online/tasks.py index 69d1e915..dd69be55 100644 --- a/apps/quickbooks_online/tasks.py +++ b/apps/quickbooks_online/tasks.py @@ -5,17 +5,20 @@ from django.conf import settings from django.db import transaction +from django.db.models import Q from qbosdk.exceptions import WrongParamsError +from fyle_accounting_mappings.models import Mapping + from fyle_jobs import FyleJobsSDK from fyle_qbo_api.exceptions import BulkError -from apps.fyle.utils import FyleConnector from apps.fyle.models import ExpenseGroup from apps.tasks.models import TaskLog -from apps.mappings.models import EmployeeMapping, GeneralMapping, CategoryMapping +from apps.mappings.models import GeneralMapping from apps.workspaces.models import QBOCredential, FyleCredential +from apps.fyle.utils import FyleConnector from .models import Bill, BillLineitem, Cheque, ChequeLineitem, CreditCardPurchase, CreditCardPurchaseLineitem,\ JournalEntry, JournalEntryLineitem @@ -43,7 +46,7 @@ def schedule_bills_creation(workspace_id: int, expense_group_ids: List[str], use fyle_credentials = FyleCredential.objects.get( workspace_id=workspace_id) - fyle_connector = FyleConnector(fyle_credentials.refresh_token) + fyle_connector = FyleConnector(fyle_credentials.refresh_token, workspace_id) fyle_sdk_connection = fyle_connector.connection jobs = FyleJobsSDK(settings.FYLE_JOBS_URL, fyle_sdk_connection) @@ -80,7 +83,7 @@ def create_bill(expense_group, task_log): qbo_credentials = QBOCredential.objects.get(workspace_id=expense_group.workspace_id) - qbo_connection = QBOConnector(qbo_credentials) + qbo_connection = QBOConnector(qbo_credentials, expense_group.workspace_id) created_bill = qbo_connection.post_bill(bill_object, bill_lineitems_objects) @@ -134,6 +137,7 @@ def create_bill(expense_group, task_log): def __validate_expense_group(expense_group: ExpenseGroup): bulk_errors = [] row = 0 + try: GeneralMapping.objects.get(workspace_id=expense_group.workspace_id) except GeneralMapping.DoesNotExist: @@ -146,11 +150,13 @@ def __validate_expense_group(expense_group: ExpenseGroup): }) try: - EmployeeMapping.objects.get( - employee_email=expense_group.description.get('employee_email'), + Mapping.objects.get( + Q(destination_type='VENDOR') | Q(destination_type='EMPLOYEE'), + source_type='EMPLOYEE', + source__value=expense_group.description.get('employee_email'), workspace_id=expense_group.workspace_id ) - except EmployeeMapping.DoesNotExist: + except Mapping.DoesNotExist: bulk_errors.append({ 'row': None, 'expense_group_id': expense_group.id, @@ -162,14 +168,19 @@ def __validate_expense_group(expense_group: ExpenseGroup): expenses = expense_group.expenses.all() for lineitem in expenses: - account = CategoryMapping.objects.filter(category=lineitem.category, - sub_category=lineitem.sub_category, - workspace_id=expense_group.workspace_id).first() + category = lineitem.category if lineitem.category == lineitem.sub_category else '{0} / {1}'.format( + lineitem.category, lineitem.sub_category) + + account = Mapping.objects.filter( + source_type='CATEGORY', + source__value=category, + workspace_id=expense_group.workspace_id + ).first() if not account: bulk_errors.append({ 'row': row, 'expense_group_id': expense_group.id, - 'value': '{0} / {1}'.format(lineitem.category, lineitem.sub_category), + 'value': category, 'type': 'Category Mapping', 'message': 'Category Mapping not found' }) @@ -232,7 +243,7 @@ def create_cheque(expense_group, task_log): qbo_credentials = QBOCredential.objects.get(workspace_id=expense_group.workspace_id) - qbo_connection = QBOConnector(qbo_credentials) + qbo_connection = QBOConnector(qbo_credentials, expense_group.workspace_id) created_cheque = qbo_connection.post_cheque(cheque_object, cheque_line_item_objects) @@ -339,7 +350,7 @@ def create_credit_card_purchase(expense_group, task_log): ) qbo_credentials = QBOCredential.objects.get(workspace_id=expense_group.workspace_id) - qbo_connection = QBOConnector(qbo_credentials) + qbo_connection = QBOConnector(qbo_credentials, expense_group.workspace_id) created_credit_card_purchase = qbo_connection.post_credit_card_purchase( credit_card_purchase_object, credit_card_purchase_lineitems_objects @@ -448,7 +459,7 @@ def create_journal_entry(expense_group, task_log): qbo_credentials = QBOCredential.objects.get(workspace_id=expense_group.workspace_id) - qbo_connection = QBOConnector(qbo_credentials) + qbo_connection = QBOConnector(qbo_credentials, expense_group.workspace_id) created_journal_entry = qbo_connection.post_journal_entry(journal_entry_object, journal_entry_lineitems_objects) diff --git a/apps/quickbooks_online/urls.py b/apps/quickbooks_online/urls.py index f30631c8..9d002c9e 100644 --- a/apps/quickbooks_online/urls.py +++ b/apps/quickbooks_online/urls.py @@ -14,17 +14,22 @@ """ from django.urls import path -from .views import VendorView, EmployeeView, AccountView, ClassView, DepartmentView, BillView, BillScheduleView, \ - CustomerView, ChequeScheduleView, ChequeView, CreditCardPurchaseView, CreditCardPurchaseScheduleView,\ - JournalEntryView, JournalEntryScheduleView +from .views import VendorView, EmployeeView, AccountView, CreditCardAccountView, ClassView, DepartmentView, BillView, \ + BillScheduleView, CustomerView, ChequeScheduleView, ChequeView, CreditCardPurchaseView, \ + CreditCardPurchaseScheduleView, JournalEntryView, JournalEntryScheduleView, BankAccountView, AccountsPayableView, \ + PreferencesView urlpatterns = [ - path('vendors/', VendorView.as_view({'get': 'get_vendors'})), - path('employees/', EmployeeView.as_view({'get': 'get_employees'})), - path('accounts/', AccountView.as_view({'get': 'get_accounts'})), - path('classes/', ClassView.as_view({'get': 'get_classes'})), - path('departments/', DepartmentView.as_view({'get': 'get_departments'})), - path('customers/', CustomerView.as_view({'get': 'get_customers'})), + path('preferences/', PreferencesView.as_view()), + path('vendors/', VendorView.as_view()), + path('employees/', EmployeeView.as_view()), + path('accounts/', AccountView.as_view()), + path('credit_card_accounts/', CreditCardAccountView.as_view()), + path('bank_accounts/', BankAccountView.as_view()), + path('accounts_payables/', AccountsPayableView.as_view()), + path('classes/', ClassView.as_view()), + path('departments/', DepartmentView.as_view()), + path('customers/', CustomerView.as_view()), path('bills/', BillView.as_view()), path('bills/trigger/', BillScheduleView.as_view()), path('checks/', ChequeView.as_view()), diff --git a/apps/quickbooks_online/utils.py b/apps/quickbooks_online/utils.py index 21021306..40756b5e 100644 --- a/apps/quickbooks_online/utils.py +++ b/apps/quickbooks_online/utils.py @@ -5,6 +5,7 @@ from qbosdk import QuickbooksOnlineSDK from apps.workspaces.models import QBOCredential +from fyle_accounting_mappings.models import DestinationAttribute from .models import BillLineitem, Bill, ChequeLineitem, Cheque, CreditCardPurchase, CreditCardPurchaseLineitem, \ JournalEntry, JournalEntryLineitem @@ -14,7 +15,7 @@ class QBOConnector: """ QBO utility functions """ - def __init__(self, credentials_object: QBOCredential): + def __init__(self, credentials_object: QBOCredential, workspace_id: int): client_id = settings.QBO_CLIENT_ID client_secret = settings.QBO_CLIENT_SECRET environment = settings.QBO_ENVIRONMENT @@ -27,44 +28,145 @@ def __init__(self, credentials_object: QBOCredential): environment=environment ) + self.workspace_id = workspace_id + credentials_object.refresh_token = self.connection.refresh_token credentials_object.save() - def get_accounts(self): + def sync_accounts(self, account_type: str): """ Get accounts """ - return self.connection.accounts.get() + accounts = self.connection.accounts.get() + + accounts = list(filter(lambda current_account: current_account['AccountType'] == account_type, accounts)) + + account_attributes = [] + + if account_type == 'Expense': + attribute_type = 'ACCOUNT' + display_name = 'Account' + elif account_type == 'Credit Card': + attribute_type = 'CREDIT_CARD_ACCOUNT' + display_name = 'Credit Card Account' + elif account_type == 'Bank': + attribute_type = 'BANK_ACCOUNT' + display_name = 'Bank Account' + else: + attribute_type = 'ACCOUNTS_PAYABLE' + display_name = 'Accounts Payable' - def get_departments(self): + for account in accounts: + account_attributes.append({ + 'attribute_type': attribute_type, + 'display_name': display_name, + 'value': account['Name'], + 'destination_id': account['Id'] + }) + + account_attributes = DestinationAttribute.bulk_upsert_destination_attributes( + account_attributes, self.workspace_id) + return account_attributes + + def sync_departments(self): """ Get departments """ - return self.connection.departments.get() + departments = self.connection.departments.get() + + department_attributes = [] + + for department in departments: + department_attributes.append({ + 'attribute_type': 'DEPARTMENT', + 'display_name': 'Department', + 'value': department['Name'], + 'destination_id': department['Id'] + }) - def get_vendors(self): + account_attributes = DestinationAttribute.bulk_upsert_destination_attributes( + department_attributes, self.workspace_id) + return account_attributes + + def sync_vendors(self): """ Get vendors """ - return self.connection.vendors.get() + vendors = self.connection.vendors.get() + + vendor_attributes = [] + + for vendor in vendors: + vendor_attributes.append({ + 'attribute_type': 'VENDOR', + 'display_name': 'vendor', + 'value': vendor['DisplayName'], + 'destination_id': vendor['Id'] + }) + + account_attributes = DestinationAttribute.bulk_upsert_destination_attributes( + vendor_attributes, self.workspace_id) + return account_attributes - def get_employees(self): + def sync_employees(self): """ Get employees """ - return self.connection.employees.get() + employees = self.connection.employees.get() - def get_classes(self): + employee_attributes = [] + + for employee in employees: + employee_attributes.append({ + 'attribute_type': 'EMPLOYEE', + 'display_name': 'employee', + 'value': employee['DisplayName'], + 'destination_id': employee['Id'] + }) + + account_attributes = DestinationAttribute.bulk_upsert_destination_attributes( + employee_attributes, self.workspace_id) + return account_attributes + + def sync_classes(self): """ Get classes """ - return self.connection.classes.get() + classes = self.connection.classes.get() + + class_attributes = [] - def get_customers(self): + for qbo_class in classes: + class_attributes.append({ + 'attribute_type': 'CLASS', + 'display_name': 'class', + 'value': qbo_class['Name'], + 'destination_id': qbo_class['Id'] + }) + + account_attributes = DestinationAttribute.bulk_upsert_destination_attributes( + class_attributes, self.workspace_id) + return account_attributes + + def sync_customers(self): """ - Get classes + Get customers """ - return self.connection.customers.get() + customers = self.connection.customers.get() + + customer_attributes = [] + + for customer in customers: + customer_attributes.append({ + 'attribute_type': 'CUSTOMER', + 'display_name': 'customer', + 'value': customer['DisplayName'], + 'destination_id': customer['Id'] + }) + + account_attributes = DestinationAttribute.bulk_upsert_destination_attributes( + customer_attributes, self.workspace_id) + return account_attributes @staticmethod def purchase_object_payload(purchase_object, line, payment_type, account_ref, doc_number): @@ -87,8 +189,7 @@ def purchase_object_payload(purchase_object, line, payment_type, account_ref, do "value": purchase_object.currency }, 'PrivateNote': purchase_object.private_note, - 'Line': line, - 'DocNumber': doc_number + 'Line': line } return purchase_object_payload @@ -143,8 +244,7 @@ def __construct_bill(self, bill: Bill, bill_lineitems: List[BillLineitem]) -> Di "value": bill.currency }, 'PrivateNote': bill.private_note, - 'Line': self.__construct_bill_lineitems(bill_lineitems), - 'DocNumber': bill.bill_number + 'Line': self.__construct_bill_lineitems(bill_lineitems) } return bill_payload @@ -176,7 +276,10 @@ def __construct_cheque_lineitems(cheque_lineitems: List[ChequeLineitem]) -> List }, 'ClassRef': { 'value': line.class_id - } + }, + 'CustomerRef': { + 'value': line.customer_id + }, } } lines.append(line) @@ -323,8 +426,7 @@ def __construct_journal_entry(self, journal_entry: JournalEntry, 'Line': lines, 'CurrencyRef': { "value": journal_entry.currency - }, - 'DocNumber': journal_entry.journal_entry_number + } } return journal_entry_payload @@ -335,3 +437,10 @@ def post_journal_entry(self, journal_entry: JournalEntry, journal_entry_lineitem journal_entry_payload = self.__construct_journal_entry(journal_entry, journal_entry_lineitems) created_journal_entry = self.connection.journal_entries.post(journal_entry_payload) return created_journal_entry + + def get_company_preference(self): + """ + Get QBO company preferences + :return: + """ + return self.connection.preferences.get() diff --git a/apps/quickbooks_online/views.py b/apps/quickbooks_online/views.py index 074945f8..3e3f4f60 100644 --- a/apps/quickbooks_online/views.py +++ b/apps/quickbooks_online/views.py @@ -1,8 +1,12 @@ -from rest_framework import viewsets from rest_framework.response import Response from rest_framework.views import status from rest_framework import generics +from qbosdk.exceptions import WrongParamsError + +from fyle_accounting_mappings.models import DestinationAttribute +from fyle_accounting_mappings.serializers import DestinationAttributeSerializer + from fyle_qbo_api.utils import assert_valid from apps.fyle.models import ExpenseGroup @@ -17,25 +21,30 @@ from .serializers import BillSerializer, ChequeSerializer, CreditCardPurchaseSerializer, JournalEntrySerializer -class VendorView(viewsets.ViewSet): +class VendorView(generics.ListCreateAPIView): """ Vendor view """ + serializer_class = DestinationAttributeSerializer + pagination_class = None + + def get_queryset(self): + return DestinationAttribute.objects.filter( + attribute_type='VENDOR', workspace_id=self.kwargs['workspace_id']).order_by('value') - def get_vendors(self, request, **kwargs): + def post(self, request, *args, **kwargs): """ Get vendors from QBO """ try: - qbo_credentials = QBOCredential.objects.get( - workspace_id=kwargs['workspace_id']) + qbo_credentials = QBOCredential.objects.get(workspace_id=kwargs['workspace_id']) - qbo_connector = QBOConnector(qbo_credentials) + qbo_connector = QBOConnector(qbo_credentials, workspace_id=kwargs['workspace_id']) - vendors = qbo_connector.get_vendors() + vendors = qbo_connector.sync_vendors() return Response( - data=vendors, + data=self.serializer_class(vendors, many=True).data, status=status.HTTP_200_OK ) except QBOCredential.DoesNotExist: @@ -47,25 +56,135 @@ def get_vendors(self, request, **kwargs): ) -class EmployeeView(viewsets.ViewSet): +class EmployeeView(generics.ListCreateAPIView): """ Employee view """ + serializer_class = DestinationAttributeSerializer + pagination_class = None + + def get_queryset(self): + return DestinationAttribute.objects.filter( + attribute_type='EMPLOYEE', workspace_id=self.kwargs['workspace_id']).order_by('value') - def get_employees(self, request, **kwargs): + def post(self, request, *args, **kwargs): """ Get employees from QBO """ try: - qbo_credentials = QBOCredential.objects.get( - workspace_id=kwargs['workspace_id']) + qbo_credentials = QBOCredential.objects.get(workspace_id=kwargs['workspace_id']) + + qbo_connector = QBOConnector(qbo_credentials, workspace_id=kwargs['workspace_id']) + + employees = qbo_connector.sync_employees() + + return Response( + data=self.serializer_class(employees, many=True).data, + status=status.HTTP_200_OK + ) + except QBOCredential.DoesNotExist: + return Response( + data={ + 'message': 'QBO credentials not found in workspace' + }, + status=status.HTTP_400_BAD_REQUEST + ) + + +class AccountView(generics.ListCreateAPIView): + """ + Account view + """ + serializer_class = DestinationAttributeSerializer + pagination_class = None + + def get_queryset(self): + return DestinationAttribute.objects.filter( + attribute_type='ACCOUNT', workspace_id=self.kwargs['workspace_id']).order_by('value') + + def post(self, request, *args, **kwargs): + """ + Get accounts from QBO + """ + try: + qbo_credentials = QBOCredential.objects.get(workspace_id=kwargs['workspace_id']) + + qbo_connector = QBOConnector(qbo_credentials, workspace_id=kwargs['workspace_id']) + + accounts = qbo_connector.sync_accounts(account_type='Expense') + + return Response( + data=self.serializer_class(accounts, many=True).data, + status=status.HTTP_200_OK + ) + except QBOCredential.DoesNotExist: + return Response( + data={ + 'message': 'QBO credentials not found in workspace' + }, + status=status.HTTP_400_BAD_REQUEST + ) + + +class CreditCardAccountView(generics.ListCreateAPIView): + """ + Account view + """ + serializer_class = DestinationAttributeSerializer + pagination_class = None + + def get_queryset(self): + return DestinationAttribute.objects.filter( + attribute_type='CREDIT_CARD_ACCOUNT', workspace_id=self.kwargs['workspace_id']).order_by('value') + + def post(self, request, *args, **kwargs): + """ + Get accounts from QBO + """ + try: + qbo_credentials = QBOCredential.objects.get(workspace_id=kwargs['workspace_id']) + + qbo_connector = QBOConnector(qbo_credentials, workspace_id=kwargs['workspace_id']) + + accounts = qbo_connector.sync_accounts(account_type='Credit Card') + + return Response( + data=self.serializer_class(accounts, many=True).data, + status=status.HTTP_200_OK + ) + except QBOCredential.DoesNotExist: + return Response( + data={ + 'message': 'QBO credentials not found in workspace' + }, + status=status.HTTP_400_BAD_REQUEST + ) + + +class BankAccountView(generics.ListCreateAPIView): + """ + Account view + """ + serializer_class = DestinationAttributeSerializer + pagination_class = None + + def get_queryset(self): + return DestinationAttribute.objects.filter( + attribute_type='BANK_ACCOUNT', workspace_id=self.kwargs['workspace_id']).order_by('value') + + def post(self, request, *args, **kwargs): + """ + Get accounts from QBO + """ + try: + qbo_credentials = QBOCredential.objects.get(workspace_id=kwargs['workspace_id']) - qbo_connector = QBOConnector(qbo_credentials) + qbo_connector = QBOConnector(qbo_credentials, workspace_id=kwargs['workspace_id']) - employees = qbo_connector.get_employees() + accounts = qbo_connector.sync_accounts(account_type='Bank') return Response( - data=employees, + data=self.serializer_class(accounts, many=True).data, status=status.HTTP_200_OK ) except QBOCredential.DoesNotExist: @@ -77,25 +196,30 @@ def get_employees(self, request, **kwargs): ) -class AccountView(viewsets.ViewSet): +class AccountsPayableView(generics.ListCreateAPIView): """ Account view """ + serializer_class = DestinationAttributeSerializer + pagination_class = None - def get_accounts(self, request, **kwargs): + def get_queryset(self): + return DestinationAttribute.objects.filter( + attribute_type='ACCOUNTS_PAYABLE', workspace_id=self.kwargs['workspace_id']).order_by('value') + + def post(self, request, *args, **kwargs): """ Get accounts from QBO """ try: - qbo_credentials = QBOCredential.objects.get( - workspace_id=kwargs['workspace_id']) + qbo_credentials = QBOCredential.objects.get(workspace_id=kwargs['workspace_id']) - qbo_connector = QBOConnector(qbo_credentials) + qbo_connector = QBOConnector(qbo_credentials, workspace_id=kwargs['workspace_id']) - accounts = qbo_connector.get_accounts() + accounts = qbo_connector.sync_accounts(account_type='Accounts Payable') return Response( - data=accounts, + data=self.serializer_class(accounts, many=True).data, status=status.HTTP_200_OK ) except QBOCredential.DoesNotExist: @@ -107,25 +231,31 @@ def get_accounts(self, request, **kwargs): ) -class ClassView(viewsets.ViewSet): +class ClassView(generics.ListCreateAPIView): """ Class view """ - def get_classes(self, request, **kwargs): + serializer_class = DestinationAttributeSerializer + pagination_class = None + + def get_queryset(self): + return DestinationAttribute.objects.filter( + attribute_type='CLASS', workspace_id=self.kwargs['workspace_id']).order_by('value') + + def post(self, request, *args, **kwargs): """ Get classes from QBO """ try: - qbo_credentials = QBOCredential.objects.get( - workspace_id=kwargs['workspace_id']) + qbo_credentials = QBOCredential.objects.get(workspace_id=kwargs['workspace_id']) - qbo_connector = QBOConnector(qbo_credentials) + qbo_connector = QBOConnector(qbo_credentials, workspace_id=kwargs['workspace_id']) - classes = qbo_connector.get_classes() + classes = qbo_connector.sync_classes() return Response( - data=classes, + data=self.serializer_class(classes, many=True).data, status=status.HTTP_200_OK ) except QBOCredential.DoesNotExist: @@ -137,25 +267,62 @@ def get_classes(self, request, **kwargs): ) -class DepartmentView(viewsets.ViewSet): +class PreferencesView(generics.RetrieveAPIView): + """ + Preferences View + """ + def get(self, request, *args, **kwargs): + try: + qbo_credentials = QBOCredential.objects.get(workspace_id=kwargs['workspace_id']) + + qbo_connector = QBOConnector(qbo_credentials, workspace_id=kwargs['workspace_id']) + + preferences = qbo_connector.get_company_preference() + + return Response( + data=preferences, + status=status.HTTP_200_OK + ) + except QBOCredential.DoesNotExist: + return Response( + data={ + 'message': 'QBO credentials not found in workspace' + }, + status=status.HTTP_400_BAD_REQUEST + ) + except WrongParamsError: + return Response( + data={ + 'message': 'Quickbooks Online connection expired' + }, + status=status.HTTP_400_BAD_REQUEST + ) + + +class DepartmentView(generics.ListCreateAPIView): """ Department view """ + serializer_class = DestinationAttributeSerializer + pagination_class = None - def get_departments(self, request, **kwargs): + def get_queryset(self): + return DestinationAttribute.objects.filter( + attribute_type='DEPARTMENT', workspace_id=self.kwargs['workspace_id']).order_by('value') + + def post(self, request, *args, **kwargs): """ Get departments from QBO """ try: - qbo_credentials = QBOCredential.objects.get( - workspace_id=kwargs['workspace_id']) + qbo_credentials = QBOCredential.objects.get(workspace_id=kwargs['workspace_id']) - qbo_connector = QBOConnector(qbo_credentials) + qbo_connector = QBOConnector(qbo_credentials, workspace_id=kwargs['workspace_id']) - departments = qbo_connector.get_departments() + departments = qbo_connector.sync_departments() return Response( - data=departments, + data=self.serializer_class(departments, many=True).data, status=status.HTTP_200_OK ) except QBOCredential.DoesNotExist: @@ -167,25 +334,31 @@ def get_departments(self, request, **kwargs): ) -class CustomerView(viewsets.ViewSet): +class CustomerView(generics.ListCreateAPIView): """ Department view """ - def get_customers(self, request, **kwargs): + serializer_class = DestinationAttributeSerializer + pagination_class = None + + def get_queryset(self): + return DestinationAttribute.objects.filter( + attribute_type='CUSTOMER', workspace_id=self.kwargs['workspace_id']).order_by('value') + + def post(self, request, *args, **kwargs): """ - Get departments from QBO + Get customers from QBO """ try: - qbo_credentials = QBOCredential.objects.get( - workspace_id=kwargs['workspace_id']) + qbo_credentials = QBOCredential.objects.get(workspace_id=kwargs['workspace_id']) - qbo_connector = QBOConnector(qbo_credentials) + qbo_connector = QBOConnector(qbo_credentials, workspace_id=kwargs['workspace_id']) - customers = qbo_connector.get_customers() + customers = qbo_connector.sync_customers() return Response( - data=customers, + data=self.serializer_class(customers, many=True).data, status=status.HTTP_200_OK ) except QBOCredential.DoesNotExist: diff --git a/apps/users/views.py b/apps/users/views.py index 5b75d527..f3c61065 100644 --- a/apps/users/views.py +++ b/apps/users/views.py @@ -15,7 +15,6 @@ def get(self, request, *args, **kwargs): """ Get User Details """ - print(request.user) fyle_credentials = AuthToken.objects.get(user__user_id=request.user) fyle_connector = FyleConnector(fyle_credentials.refresh_token) diff --git a/apps/workspaces/tasks.py b/apps/workspaces/tasks.py index ec4aa1f0..a8910cc5 100644 --- a/apps/workspaces/tasks.py +++ b/apps/workspaces/tasks.py @@ -46,7 +46,7 @@ def schedule_sync(workspace_id: int, schedule_enabled: bool, hours: int, next_ru fyle_credentials = FyleCredential.objects.get( workspace_id=workspace_id) - fyle_connector = FyleConnector(fyle_credentials.refresh_token) + fyle_connector = FyleConnector(fyle_credentials.refresh_token, workspace_id) fyle_sdk_connection = fyle_connector.connection jobs = FyleJobsSDK(settings.FYLE_JOBS_URL, fyle_sdk_connection) @@ -72,9 +72,8 @@ def schedule_sync(workspace_id: int, schedule_enabled: bool, hours: int, next_ru def create_schedule_job(workspace_id: int, schedule: WorkspaceSchedule, user: str, start_datetime: datetime, hours: int): - fyle_credentials = FyleCredential.objects.get( - workspace_id=workspace_id) - fyle_connector = FyleConnector(fyle_credentials.refresh_token) + fyle_credentials = FyleCredential.objects.get(workspace_id=workspace_id) + fyle_connector = FyleConnector(fyle_credentials.refresh_token, workspace_id) fyle_sdk_connection = fyle_connector.connection jobs = FyleJobsSDK(settings.FYLE_JOBS_URL, fyle_sdk_connection) diff --git a/apps/workspaces/utils.py b/apps/workspaces/utils.py index ce85edb8..5adb3b79 100644 --- a/apps/workspaces/utils.py +++ b/apps/workspaces/utils.py @@ -65,17 +65,14 @@ def create_or_update_general_settings(general_settings_payload: Dict, workspace_ 'reimbursable_expenses_object' in general_settings_payload and general_settings_payload[ 'reimbursable_expenses_object'], 'reimbursable_expenses_object field is blank') - assert_valid('employee_field_mapping' in general_settings_payload and general_settings_payload[ - 'employee_field_mapping'], 'employee_field_mapping field is blank') - general_settings, _ = WorkspaceGeneralSettings.objects.update_or_create( workspace_id=workspace_id, defaults={ 'reimbursable_expenses_object': general_settings_payload['reimbursable_expenses_object'], - 'corporate_credit_card_expenses_object': general_settings_payload['corporate_credit_card_expenses_object'] - if general_settings_payload - ['corporate_credit_card_expenses_object'] else None, - 'employee_field_mapping': general_settings_payload['employee_field_mapping'] + 'corporate_credit_card_expenses_object': + general_settings_payload['corporate_credit_card_expenses_object'] + if 'corporate_credit_card_expenses_object' in general_settings_payload + and general_settings_payload['corporate_credit_card_expenses_object'] else None, } ) return general_settings diff --git a/fyle_accounting_mappings/__init__.py b/fyle_accounting_mappings/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/fyle_accounting_mappings/admin.py b/fyle_accounting_mappings/admin.py new file mode 100644 index 00000000..faea97ab --- /dev/null +++ b/fyle_accounting_mappings/admin.py @@ -0,0 +1,7 @@ +from django.contrib import admin +from .models import ExpenseAttribute, DestinationAttribute, MappingSetting, Mapping + +admin.site.register(ExpenseAttribute) +admin.site.register(DestinationAttribute) +admin.site.register(MappingSetting) +admin.site.register(Mapping) diff --git a/fyle_accounting_mappings/apps.py b/fyle_accounting_mappings/apps.py new file mode 100644 index 00000000..64420000 --- /dev/null +++ b/fyle_accounting_mappings/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class FyleAccountingMappingsConfig(AppConfig): + name = 'fyle_accounting_mappings' diff --git a/fyle_accounting_mappings/exceptions.py b/fyle_accounting_mappings/exceptions.py new file mode 100644 index 00000000..1bd7ec3b --- /dev/null +++ b/fyle_accounting_mappings/exceptions.py @@ -0,0 +1,21 @@ +""" +Exceptions +""" + + +class BulkError(Exception): + """ + Bulk Error Exception. + + Parameters: + msg (str): Short description of the error. + response: Error response. + """ + + def __init__(self, msg, response=None): + super(BulkError, self).__init__(msg) + self.message = msg + self.response = response + + def __str__(self): + return repr(self.message) diff --git a/fyle_accounting_mappings/migrations/0001_initial.py b/fyle_accounting_mappings/migrations/0001_initial.py new file mode 100644 index 00000000..9e0c9bfe --- /dev/null +++ b/fyle_accounting_mappings/migrations/0001_initial.py @@ -0,0 +1,69 @@ +# Generated by Django 3.0.3 on 2020-05-20 09:46 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('workspaces', '0003_auto_20200506_0739'), + ] + + operations = [ + migrations.CreateModel( + name='DestinationAttribute', + fields=[ + ('id', models.AutoField(primary_key=True, serialize=False)), + ('attribute_type', models.CharField(help_text='Type of expense attribute', max_length=255)), + ('display_name', models.CharField(help_text='Display name of attribute', max_length=255)), + ('value', models.CharField(help_text='Value of expense attribute', max_length=255)), + ('destination_id', models.CharField(help_text='Destination ID', max_length=255)), + ('created_at', models.DateTimeField(auto_now_add=True, help_text='Created at datetime')), + ('updated_at', models.DateTimeField(auto_now=True, help_text='Updated at datetime')), + ('workspace', models.ForeignKey(help_text='Reference to Workspace model', on_delete=django.db.models.deletion.PROTECT, to='workspaces.Workspace')), + ], + ), + migrations.CreateModel( + name='ExpenseAttribute', + fields=[ + ('id', models.AutoField(primary_key=True, serialize=False)), + ('attribute_type', models.CharField(help_text='Type of expense attribute', max_length=255)), + ('display_name', models.CharField(help_text='Display name of expense attribute', max_length=255)), + ('value', models.CharField(help_text='Value of expense attribute', max_length=255)), + ('source_id', models.CharField(help_text='Fyle ID', max_length=255)), + ('created_at', models.DateTimeField(auto_now_add=True, help_text='Created at datetime')), + ('updated_at', models.DateTimeField(auto_now=True, help_text='Updated at datetime')), + ('workspace', models.ForeignKey(help_text='Reference to Workspace model', on_delete=django.db.models.deletion.PROTECT, to='workspaces.Workspace')), + ], + ), + migrations.CreateModel( + name='MappingSetting', + fields=[ + ('id', models.AutoField(primary_key=True, serialize=False)), + ('source_field', models.CharField(help_text='Source mapping field', max_length=255)), + ('destination_field', models.CharField(help_text='Destination mapping field', max_length=40, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True, help_text='Created at datetime')), + ('updated_at', models.DateTimeField(auto_now=True, help_text='Updated at datetime')), + ('workspace', models.ForeignKey(help_text='Reference to Workspace model', on_delete=django.db.models.deletion.PROTECT, to='workspaces.Workspace')), + ], + ), + migrations.CreateModel( + name='Mapping', + fields=[ + ('id', models.AutoField(primary_key=True, serialize=False)), + ('source_type', models.CharField(help_text='Fyle Enum', max_length=255)), + ('destination_type', models.CharField(help_text='Destination Enum', max_length=255)), + ('created_at', models.DateTimeField(auto_now_add=True, help_text='Created at datetime')), + ('updated_at', models.DateTimeField(auto_now=True, help_text='Updated at datetime')), + ('destination', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='fyle_accounting_mappings.DestinationAttribute')), + ('source', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='fyle_accounting_mappings.ExpenseAttribute')), + ('workspace', models.ForeignKey(help_text='Reference to Workspace model', on_delete=django.db.models.deletion.PROTECT, to='workspaces.Workspace')), + ], + options={ + 'unique_together': {('source_type', 'source', 'destination_type', 'workspace')}, + }, + ), + ] diff --git a/fyle_accounting_mappings/migrations/__init__.py b/fyle_accounting_mappings/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/fyle_accounting_mappings/models.py b/fyle_accounting_mappings/models.py new file mode 100644 index 00000000..20f76baf --- /dev/null +++ b/fyle_accounting_mappings/models.py @@ -0,0 +1,189 @@ +import importlib +from typing import List, Dict + +from django.db import models, transaction + +from .exceptions import BulkError +from .utils import assert_valid + +workspace_models = importlib.import_module("apps.workspaces.models") +Workspace = workspace_models.Workspace + + +def validate_mapping_settings(mappings_settings: List[Dict]): + bulk_errors = [] + + row = 0 + + for mappings_setting in mappings_settings: + if ('source_field' not in mappings_setting) and (not mappings_setting['source_field']): + bulk_errors.append({ + 'row': row, + 'value': None, + 'message': 'source field cannot be empty' + }) + + if ('destination_field' not in mappings_setting) and (not mappings_setting['destination_field']): + bulk_errors.append({ + 'row': row, + 'value': None, + 'message': 'destination field cannot be empty' + }) + + row = row + 1 + + if bulk_errors: + raise BulkError('Errors while creating settings', bulk_errors) + + +class ExpenseAttribute(models.Model): + """ + Fyle Expense Attributes + """ + id = models.AutoField(primary_key=True) + attribute_type = models.CharField(max_length=255, help_text='Type of expense attribute') + display_name = models.CharField(max_length=255, help_text='Display name of expense attribute') + value = models.CharField(max_length=255, help_text='Value of expense attribute') + source_id = models.CharField(max_length=255, help_text='Fyle ID') + workspace = models.ForeignKey(Workspace, on_delete=models.PROTECT, help_text='Reference to Workspace model') + created_at = models.DateTimeField(auto_now_add=True, help_text='Created at datetime') + updated_at = models.DateTimeField(auto_now=True, help_text='Updated at datetime') + + @staticmethod + def bulk_upsert_expense_attributes(attributes: List[Dict], workspace_id): + """ + Get or create expense attributes + """ + expense_attributes = [] + + with transaction.atomic(): + for attribute in attributes: + expense_attribute, _ = ExpenseAttribute.objects.update_or_create( + attribute_type=attribute['attribute_type'], + value=attribute['value'], + workspace_id=workspace_id, + defaults={ + 'source_id': attribute['source_id'], + 'display_name': attribute['display_name'], + } + ) + expense_attributes.append(expense_attribute) + return expense_attributes + + +class DestinationAttribute(models.Model): + """ + Destination Expense Attributes + """ + id = models.AutoField(primary_key=True) + attribute_type = models.CharField(max_length=255, help_text='Type of expense attribute') + display_name = models.CharField(max_length=255, help_text='Display name of attribute') + value = models.CharField(max_length=255, help_text='Value of expense attribute') + destination_id = models.CharField(max_length=255, help_text='Destination ID') + workspace = models.ForeignKey(Workspace, on_delete=models.PROTECT, help_text='Reference to Workspace model') + created_at = models.DateTimeField(auto_now_add=True, help_text='Created at datetime') + updated_at = models.DateTimeField(auto_now=True, help_text='Updated at datetime') + + @staticmethod + def bulk_upsert_destination_attributes(attributes: List[Dict], workspace_id): + """ + get or create destination attributes + """ + destination_attributes = [] + with transaction.atomic(): + for attribute in attributes: + destination_attribute, _ = DestinationAttribute.objects.update_or_create( + attribute_type=attribute['attribute_type'], + value=attribute['value'], + workspace_id=workspace_id, + defaults={ + 'display_name': attribute['display_name'], + 'destination_id': attribute['destination_id'] + } + ) + destination_attributes.append(destination_attribute) + return destination_attributes + + +class MappingSetting(models.Model): + """ + Mapping Settings + """ + id = models.AutoField(primary_key=True) + source_field = models.CharField(max_length=255, help_text='Source mapping field') + destination_field = models.CharField(max_length=40, help_text='Destination mapping field', null=True) + workspace = models.ForeignKey(Workspace, on_delete=models.PROTECT, help_text='Reference to Workspace model') + created_at = models.DateTimeField(auto_now_add=True, help_text='Created at datetime') + updated_at = models.DateTimeField(auto_now=True, help_text='Updated at datetime') + + @staticmethod + def bulk_upsert_mapping_setting(settings: List[Dict], workspace_id: int): + """ + Bulk update or create mapping setting + """ + validate_mapping_settings(settings) + mapping_settings = [] + + with transaction.atomic(): + for setting in settings: + mapping_setting, _ = MappingSetting.objects.get_or_create( + source_field=setting['source_field'], + workspace_id=workspace_id, + destination_field=setting['destination_field'] + ) + mapping_settings.append(mapping_setting) + + return mapping_settings + + +class Mapping(models.Model): + """ + Mappings + """ + id = models.AutoField(primary_key=True) + source_type = models.CharField(max_length=255, help_text='Fyle Enum') + destination_type = models.CharField(max_length=255, help_text='Destination Enum') + source = models.ForeignKey(ExpenseAttribute, on_delete=models.PROTECT) + destination = models.ForeignKey(DestinationAttribute, on_delete=models.PROTECT) + workspace = models.ForeignKey(Workspace, on_delete=models.PROTECT, help_text='Reference to Workspace model') + created_at = models.DateTimeField(auto_now_add=True, help_text='Created at datetime') + updated_at = models.DateTimeField(auto_now=True, help_text='Updated at datetime') + + class Meta: + unique_together = ('source_type', 'source', 'destination_type', 'workspace') + + @staticmethod + def create_or_update_mapping(source_type: str, destination_type: str, + source_value: str, destination_value: str, workspace_id: int): + """ + Bulk update or create mappings + source_type = 'Type of Source attribute, eg. CATEGORY', + destination_type = 'Type of Destination attribute, eg. ACCOUNT', + source_value = 'Source value to be mapped, eg. category name', + destination_value = 'Destination value to be mapped, eg. account name' + workspace_id = Unique Workspace id + """ + settings = MappingSetting.objects.filter(source_field=source_type, destination_field=destination_type, + workspace_id=workspace_id).first() + + assert_valid( + settings is not None and settings != [], + 'Settings for Destination {0} / Source {1} not found'.format(destination_type, source_type) + ) + + mapping, _ = Mapping.objects.update_or_create( + source_type=source_type, + source=ExpenseAttribute.objects.get( + attribute_type=source_type, value=source_value, workspace_id=workspace_id + ) if source_value else None, + destination_type=destination_type, + workspace=Workspace.objects.get(pk=workspace_id), + defaults={ + 'destination': DestinationAttribute.objects.get( + attribute_type=destination_type, + value=destination_value, + workspace_id=workspace_id + ) + } + ) + return mapping diff --git a/fyle_accounting_mappings/serializers.py b/fyle_accounting_mappings/serializers.py new file mode 100644 index 00000000..6fd911ae --- /dev/null +++ b/fyle_accounting_mappings/serializers.py @@ -0,0 +1,44 @@ +""" +Mapping Serializers +""" +from rest_framework import serializers +from .models import ExpenseAttribute, DestinationAttribute, MappingSetting, Mapping + + +class ExpenseAttributeSerializer(serializers.ModelSerializer): + """ + Expense Attribute serializer + """ + class Meta: + model = ExpenseAttribute + fields = '__all__' + + +class DestinationAttributeSerializer(serializers.ModelSerializer): + """ + Destination Attribute serializer + """ + class Meta: + model = DestinationAttribute + fields = '__all__' + + +class MappingSettingSerializer(serializers.ModelSerializer): + """ + Mapping Setting serializer + """ + class Meta: + model = MappingSetting + fields = '__all__' + + +class MappingSerializer(serializers.ModelSerializer): + """ + Mapping serializer + """ + source = ExpenseAttributeSerializer() + destination = DestinationAttributeSerializer() + + class Meta: + model = Mapping + fields = '__all__' diff --git a/fyle_accounting_mappings/setup.py b/fyle_accounting_mappings/setup.py new file mode 100644 index 00000000..b46aa9f0 --- /dev/null +++ b/fyle_accounting_mappings/setup.py @@ -0,0 +1,30 @@ +""" +Project setup file +""" +import setuptools + +with open('README.md', 'r') as f: + long_description = f.read() + +setuptools.setup( + name='fyle-accounting-mappings', + version='0.1.0', + author='Shwetabh Kumar', + author_email='shwetabh.kumar@fyle.in', + description='Django application to store the accounting mappings in a generic manner', + license='MIT', + long_description=long_description, + long_description_content_type='text/markdown', + keywords=['fyle', 'rest', 'django-rest-framework', 'api', 'python', 'accounting'], + url='https://github.com/fylein/fyle-accounting-mappings', + packages=setuptools.find_packages(), + install_requires=['django>=3.0.2', 'django-rest-framework>=0.1.0'], + include_package_data=True, + classifiers=[ + 'Framework :: Django', + 'Intended Audience :: Developers', + 'Intended Audience :: System Administrators', + 'Operating System :: OS Independent', + 'Topic :: Software Development' + ] +) diff --git a/fyle_accounting_mappings/tests.py b/fyle_accounting_mappings/tests.py new file mode 100644 index 00000000..e69de29b diff --git a/fyle_accounting_mappings/urls.py b/fyle_accounting_mappings/urls.py new file mode 100644 index 00000000..864e1cae --- /dev/null +++ b/fyle_accounting_mappings/urls.py @@ -0,0 +1,23 @@ +"""fyle_accounting_mappings URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/3.0/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.urls import path + +from .views import MappingSettingsView, MappingsView + +urlpatterns = [ + path('settings/', MappingSettingsView.as_view()), + path('', MappingsView.as_view()) +] diff --git a/fyle_accounting_mappings/utils.py b/fyle_accounting_mappings/utils.py new file mode 100644 index 00000000..fe007608 --- /dev/null +++ b/fyle_accounting_mappings/utils.py @@ -0,0 +1,15 @@ +from rest_framework.views import Response +from rest_framework.serializers import ValidationError + + +def assert_valid(condition: bool, message: str) -> Response or None: + """ + Assert conditions + :param condition: Boolean condition + :param message: Bad request message + :return: Response or None + """ + if not condition: + raise ValidationError(detail={ + 'message': message + }) diff --git a/fyle_accounting_mappings/views.py b/fyle_accounting_mappings/views.py new file mode 100644 index 00000000..751d46af --- /dev/null +++ b/fyle_accounting_mappings/views.py @@ -0,0 +1,99 @@ +import logging +from typing import Dict, List + +from rest_framework.generics import ListCreateAPIView +from rest_framework.response import Response +from rest_framework.views import status + +from .exceptions import BulkError +from .utils import assert_valid +from .models import MappingSetting, Mapping, ExpenseAttribute, DestinationAttribute +from .serializers import MappingSettingSerializer, MappingSerializer + +logger = logging.getLogger(__name__) + + +class MappingSettingsView(ListCreateAPIView): + """ + Mapping Settings VIew + """ + serializer_class = MappingSettingSerializer + + def get_queryset(self): + return MappingSetting.objects.filter(workspace_id=self.kwargs['workspace_id']) + + def post(self, request, *args, **kwargs): + """ + Post mapping settings + """ + try: + mapping_settings: List[Dict] = request.data + + assert_valid(mapping_settings != [], 'Mapping settings not found') + + mapping_settings = MappingSetting.bulk_upsert_mapping_setting(mapping_settings, self.kwargs['workspace_id']) + + return Response(data=self.serializer_class(mapping_settings, many=True).data, status=status.HTTP_200_OK) + except BulkError as exception: + logger.error(exception.response) + return Response( + data=exception.response, + status=status.HTTP_400_BAD_REQUEST + ) + + +class MappingsView(ListCreateAPIView): + """ + Mapping Settings VIew + """ + serializer_class = MappingSerializer + + def get_queryset(self): + source_type = self.request.query_params.get('source_type') + + assert_valid(source_type is not None, 'query param source type not found') + + return Mapping.objects.filter(source_type=source_type, workspace_id=self.kwargs['workspace_id']) + + def post(self, request, *args, **kwargs): + """ + Post mapping settings + """ + source_type = request.data.get('source_type', None) + + assert_valid(source_type is not None, 'source type not found') + + destination_type = request.data.get('destination_type', None) + + assert_valid(destination_type is not None, 'destination type not found') + + source_value = request.data.get('source_value', None) + + destination_value = request.data.get('destination_value', None) + + assert_valid(destination_value is not None, 'destination value not found') + try: + mappings = Mapping.create_or_update_mapping( + source_type=source_type, + destination_type=destination_type, + source_value=source_value, + destination_value=destination_value, + workspace_id=self.kwargs['workspace_id'] + ) + + return Response(data=self.serializer_class(mappings).data, status=status.HTTP_200_OK) + except ExpenseAttribute.DoesNotExist: + return Response( + status=status.HTTP_400_BAD_REQUEST, + data={ + 'message': 'Fyle {0} with name {1} does not exist'.format(source_type, source_value) + } + ) + except DestinationAttribute.DoesNotExist: + return Response( + status=status.HTTP_400_BAD_REQUEST, + data={ + 'message': 'Destination {0} with name {1} does not exist'.format( + destination_type, destination_value) + } + ) diff --git a/fyle_qbo_api/settings.py b/fyle_qbo_api/settings.py index b91dd38b..086fb337 100644 --- a/fyle_qbo_api/settings.py +++ b/fyle_qbo_api/settings.py @@ -42,6 +42,8 @@ # Installed Apps 'rest_framework', 'corsheaders', + 'fyle_rest_auth', + 'fyle_accounting_mappings', # User Created Apps 'apps.users', @@ -49,8 +51,7 @@ 'apps.mappings', 'apps.fyle', 'apps.quickbooks_online', - 'apps.tasks', - 'fyle_rest_auth' + 'apps.tasks' ] MIDDLEWARE = [ diff --git a/requirements.txt b/requirements.txt index dc6da807..d0fd650c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ dj-redis-url==0.1.4 Django==3.0.3 django-cors-headers==3.2.0 django-picklefield==2.0 +django-q==1.0.2 django-request-logging==0.7.1 django-rest-framework==0.1.0 djangorestframework==3.11.0 diff --git a/scripts/001-mapping_infra.sql b/scripts/001-mapping_infra.sql new file mode 100644 index 00000000..3af991b6 --- /dev/null +++ b/scripts/001-mapping_infra.sql @@ -0,0 +1,172 @@ +-- Script to make generic mapping infra backward compatible with Quickbooks online integration +-- This script adds a bunch of settings to the settings table +-- These settings are required so that the app does not break + +rollback; +begin; + +insert into fyle_accounting_mappings_mappingsetting(source_field, destination_field, workspace_id, + created_at, updated_at) +select + 'EMPLOYEE' as source_field, + gs.employee_field_mapping as destination_field, + gs.workspace_id as workspace_id, + gs.created_at as created_at, + gs.updated_at as updated_at +from workspaces_workspacegeneralsettings gs; + +insert into fyle_accounting_mappings_mappingsetting(source_field, destination_field, workspace_id, + created_at, updated_at) +select + 'CATEGORY' as source_field, + 'ACCOUNT' as destination_field, + gs.workspace_id as workspace_id, + gs.created_at as created_at, + gs.updated_at as updated_at +from workspaces_workspacegeneralsettings gs; + +insert into fyle_accounting_mappings_mappingsetting(source_field, destination_field, workspace_id, + created_at, updated_at) +select + 'EMPLOYEE' as source_field, + 'CREDIT_CARD_ACCOUNT' as destination_field, + gs.workspace_id as workspace_id, + gs.created_at as created_at, + gs.updated_at as updated_at +from workspaces_workspacegeneralsettings gs +where gs.corporate_credit_card_expenses_object is not null; + + +insert into fyle_accounting_mappings_expenseattribute(attribute_type, display_name, value, source_id, workspace_id, + created_at, updated_at) +select + 'EMPLOYEE' as attribute_type, + 'employee' as display_name, + em.employee_email as value, + '' as source_id, + em.workspace_id as workspace_id, + em.created_at as created_at, + em.updated_at as updated_at +from mappings_employeemapping em; + +insert into fyle_accounting_mappings_expenseattribute(attribute_type, display_name, value, source_id, workspace_id, + created_at, updated_at) +select + 'CATEGORY' as attribute_type, + 'category' as display_name, + case when lower(cm.category) = lower(cm.sub_category) then + cm.category + else concat(cm.category, ' / ', cm.sub_category) end as value, + '' as source_id, + cm.workspace_id as workspace_id, + cm.created_at as created_at, + cm.updated_at as updated_at +from mappings_categorymapping cm; + +insert into fyle_accounting_mappings_destinationattribute(attribute_type, display_name, value, destination_id, + workspace_id, created_at, updated_at) +select + case when em.vendor_display_name is null then + 'EMPLOYEE' + else + 'VENDOR' end as attribute_type, + case when em.vendor_display_name is null then + 'employee' + else + 'vendor' end as display_name, + case when em.vendor_display_name is null then + em.employee_display_name + else + em.vendor_display_name end as value, + case when em.vendor_display_name is null then + em.employee_id + else + em.vendor_id end as source_id, + em.workspace_id as workspace_id, + now() as created_at, + now() as updated_at +from mappings_employeemapping em +group by em.vendor_display_name, em.vendor_id, em.employee_id, em.employee_display_name, em.workspace_id; + +insert into fyle_accounting_mappings_destinationattribute(attribute_type, display_name, value, destination_id, + workspace_id, created_at, updated_at) +select + 'CREDIT_CARD_ACCOUNT' as attribute_type, + 'Credit Card Account' as display_name, + em.ccc_account_name as value, + em.ccc_account_id as source_id, + em.workspace_id as workspace_id, + now() as created_at, + now() as updated_at +from mappings_employeemapping em +group by em.ccc_account_name, em.ccc_account_id, em.workspace_id; + +insert into fyle_accounting_mappings_destinationattribute(attribute_type, display_name, value, destination_id, + workspace_id, created_at, updated_at) +select + 'ACCOUNT' as attribute_type, + 'account' as display_name, + cm.account_name as value, + cm.account_id as destination_id, + cm.workspace_id as workspace_id, + now() as created_at, + now() as updated_at +from mappings_categorymapping cm +group by cm.account_name, cm.account_id, cm.workspace_id; + + +insert into fyle_accounting_mappings_mapping(source_type, destination_type, source_id, destination_id, workspace_id, + created_at, updated_at) +select + 'EMPLOYEE' as source_type, + case when em.vendor_display_name is null then + 'EMPLOYEE' + else + 'VENDOR' end as attribute_type, + (select id from fyle_accounting_mappings_expenseattribute ea + where em.employee_email = ea.value and ea.attribute_type = 'EMPLOYEE' + and em.workspace_id = ea.workspace_id) as source_id, + case when em.vendor_display_name is null then + (select id from fyle_accounting_mappings_destinationattribute da where em.workspace_id = da.workspace_id + and da.attribute_type = 'EMPLOYEE' and da.value = em.employee_display_name) + else + (select id from fyle_accounting_mappings_destinationattribute da where em.workspace_id = da.workspace_id + and da.attribute_type = 'VENDOR' and da.value = em.vendor_display_name) end as destination_id, + em.workspace_id as workspace_id, + em.created_at as created_at, + em.updated_at as updated_at +from mappings_employeemapping em; + + +insert into fyle_accounting_mappings_mapping(source_type, destination_type, source_id, destination_id, workspace_id, + created_at, updated_at) +select + 'CATEGORY', + 'ACCOUNT', + (select id from fyle_accounting_mappings_expenseattribute ea where + (ea.value = case when lower(cm.category) = lower(cm.sub_category) then + cm.category else concat(cm.category, ' / ', cm.sub_category) end) + and ea.workspace_id = cm.workspace_id) as source_id, + (select id from fyle_accounting_mappings_destinationattribute da where da.value = cm.account_name + and da.workspace_id = cm.workspace_id) as destination_id, + cm.workspace_id as workspace_id, + now() as created_at, + now() as updated_at +from mappings_categorymapping cm; + + +insert into fyle_accounting_mappings_mapping(source_type, destination_type, source_id, destination_id, workspace_id, + created_at, updated_at) +select + 'EMPLOYEE' as source_type, + 'CREDIT_CARD_ACCOUNT' as destination_type, + (select id from fyle_accounting_mappings_expenseattribute ea + where em.employee_email = ea.value and ea.attribute_type = 'EMPLOYEE' + and em.workspace_id = ea.workspace_id) as source_id, + (select id from fyle_accounting_mappings_destinationattribute da where + da.workspace_id = em.workspace_id and em.ccc_account_name = da.value and em.ccc_account_id = da.destination_id + and da.attribute_type = 'CREDIT_CARD_ACCOUNT') as destination_id, + em.workspace_id as workspace_id, + em.created_at as created_at, + em.updated_at as updated_at +from mappings_employeemapping em; \ No newline at end of file diff --git a/setup_template.sh b/setup_template.sh index efc59816..b0dae067 100644 --- a/setup_template.sh +++ b/setup_template.sh @@ -17,15 +17,11 @@ export FYLE_BASE_URL=FYLE BASE URL export FYLE_CLIENT_ID=FYLE CLIENT ID export FYLE_CLIENT_SECRET=FYLE CLIENT SECRET export FYLE_TOKEN_URI=FYLE TOKEN URI +export FYLE_JOBS_URL=FYLE JOBS URL # QBO Settings export QBO_CLIENT_ID=QBO CLIENT ID export QBO_CLIENT_SECRET=QBO CLIENT SECRET export QBO_REDIRECT_URI=QBO REDIRECT URI export QBO_TOKEN_URI=QBO TOKEN URI -export QBO_ENVIRONMENT=SANDBOX/PRODUCTION - -# Redis Settings -export REDIS_HOST=REDIS HOST -export REDIS_PORT=REDIS PORT -export REDIS_DB=REDIS DB \ No newline at end of file +export QBO_ENVIRONMENT=SANDBOX/PRODUCTION \ No newline at end of file