Merge pull request #92 from fivetran/bugfix/cta

Bugfix/cta-and-historical-rate-changes

.buildkite/scripts/run_models.sh

@@ -9,6 +9,7 @@ python3 -m venv venv
 . venv/bin/activate
 pip install --upgrade pip setuptools
 pip install -r integration_tests/requirements.txt
+
 mkdir -p ~/.dbt
 cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml
 

CHANGELOG.md

@@ -1,3 +1,20 @@
+# dbt_netsuite v0.11.0
+
+## 🚨 Breaking Changes 🚨:
+- This release includes a breaking change in the upstream `dbt_netsuite_source` dependency. Please refer to the respective [dbt_netsuite_source v0.8.0](https://github.com/fivetran/dbt_netsuite_source/releases/tag/v0.8.0) release notes for more information.
+
+## 🐛 Bug Fixes 🐛:
+- Adjusted our translation rate logic to calculate `converted_amount` in `netsuite__balance_sheet` and `netsuite2__balance_sheet`. 
+  - The logic is adjusted so we examine the `general_rate_type` rather than `account_category`, as is intended by Netsuite definitions.
+  - Historical and average rates now convert amounts into the `converted_amount_using_transaction_accounting_period`. Otherwise, it looks at `converted_amount_using_reporting_month`.
+  - The `is_leftside` logic is added to make sure debit values are properly assigned as negative converted values if false and positive if true.
+- Modified the Cumulative Translation Adjustment calculation within the `netsuite2__balance_sheet` model to be built upon referencing that the general_rate_type is either `historical` or `average` as opposed to checking that the account_category is `equity`.
+  - This update more accurately reflects the behavior of how the Cumulative Translation Adjustment should be calculated. The `equity` check was not as robust and had an opportunity to generate an incorrect value. 
+
+## Contributors:
+- [@jmongerlyra](https://github.com/jmongerlyra) ([#75](https://github.com/fivetran/dbt_netsuite/issues/75))
+- [@rwang-lyra](https://github.com/rwang-lyra ) ([#75](https://github.com/fivetran/dbt_netsuite/issues/75))
+
 # dbt_netsuite v0.10.0
 [PR #84](https://github.com/fivetran/dbt_netsuite/pull/84) includes the following updates:
 ## 🚨 Breaking Changes 🚨
@@ -44,7 +61,7 @@ Retained Earnings | retained_earnings
 
 # dbt_netsuite v0.9.0
 
-[PR #74](https://github.com/fivetran/dbt_netsuite/pull/74)includes the following updates:
+[PR #74](https://github.com/fivetran/dbt_netsuite/pull/74) includes the following updates:
 ## 🚨 Breaking Changes 🚨
 - Removed the `int_netsuite2__consolidated_exchange_rates` model 
   - Originally the `accounting_book_id` field was brought into the `int_netsuite2__acctxperiod_exchange_rate_map` model via `int_netsuite2__consolidated_exchange_rates`, but this was resulting in duplicate records downstream in the `netsuite2__transaction_details` model due to the way it was being joined. Now we have brought in `accounting_book_id` (accountingbook) via the `stg_netsuite2__consolidated_exchange_rates` model, so we do not have a need for `int_netsuite2__consolidated_exchange_rates` 

README.md

@@ -100,7 +100,7 @@ Include the following netsuite package version in your `packages.yml` file:
 ```yaml
 packages:
   - package: fivetran/netsuite
-    version: [">=0.10.0", "<0.11.0"]
+    version: [">=0.11.0", "<0.12.0"]
 ```
 ## Step 3: Define Netsuite.com or Netsuite2 Source
 As of April 2022 Fivetran made available a new Netsuite connector which leverages the Netsuite2 endpoint opposed to the original Netsuite.com endpoint. This package is designed to run for either or, not both. By default the `netsuite_data_model` variable for this package is set to the original `netsuite` value which runs the netsuite.com version of the package. If you would like to run the package on Netsuite2 data, you may adjust the `netsuite_data_model` variable to run the `netsuite2` version of the package.

dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'netsuite'
-version: '0.10.0'
+version: '0.11.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 
 models:

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: netsuite_integration_tests_4
+      schema: netsuite_integration_tests_5
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: netsuite_integration_tests_4
+      schema: netsuite_integration_tests_5
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: netsuite_integration_tests_4
+      schema: netsuite_integration_tests_5
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: netsuite_integration_tests_4
+      schema: netsuite_integration_tests_5
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: netsuite_integration_tests_4
+      schema: netsuite_integration_tests_5
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,11 +1,11 @@
 name: 'netsuite_integration_tests'
-version: '0.10.0'
+version: '0.11.0'
 profile: 'integration_tests'
 config-version: 2
 models:
   +materialized: table
 vars:
-  netsuite_schema: netsuite_integration_tests_4
+  netsuite_schema: netsuite_integration_tests_5
   netsuite_data_model_override: netsuite
   netsuite_source:
     # Netsuite Seed Data
@@ -57,6 +57,9 @@ seeds:
       due_date: timestamp
       startdate: timestamp
       starting: timestamp
+    netsuite2_account_data:
+      +column_types:
+        eliminate: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
 
 dispatch:
   - macro_namespace: dbt_utils

integration_tests/requirements.txt

@@ -1,7 +1,9 @@
 dbt-snowflake>=1.3.0,<2.0.0
 dbt-bigquery>=1.3.0,<2.0.0
-dbt-redshift>=1.3.0,<1.5.0
+dbt-redshift>=1.3.0,<2.0.0
 dbt-postgres>=1.3.0,<2.0.0
 dbt-spark>=1.3.0,<2.0.0
 dbt-spark[PyHive]>=1.3.0,<2.0.0
 dbt-databricks>=1.3.0,<2.0.0
+
+oscrypto @ git+https://github.com/wbond/oscrypto.git@d5f3437

integration_tests/seeds/netsuite2_account_data.csv

@@ -1,7 +1,7 @@
-id,_fivetran_deleted,_fivetran_synced,accountsearchdisplayname,accountsearchdisplaynamecopy,acctnumber,accttype,billableexpensesacct,cashflowrate,category1099misc,class,currency,custrecord_fam_account_showinfixedasset,date_deleted,deferralacct,department,description,displaynamewithhierarchy,eliminate,externalid,fullname,generalrate,includechildren,inventory,isinactive,issummary,lastmodifieddate,location,parent,reconcilewithmatching,revalue,sbankname,sbankroutingnumber,sspecacct,subsidiary
-133,FALSE,2022-04-07 7:05:11,Power,Power,,OthExpense,,AVERAGE,,,,"3, 4, 5, 6",,,,,Power,,Allowance for Doubtful Account,Power,AVERAGE,T,F,F,F,2017-05-21 1:41:07,,,F,F,,,Power,"1, 3, 4, 6"
-157,FALSE,2022-04-07 7:05:11,Levels,Levels,,OthExpense,,AVERAGE,,,,"3, 4, 5, 6",,,,,Levels,,,Levels,AVERAGE,T,F,F,F,2017-05-02 22:46:13,,,F,F,,,Levels,"1, 3, 4, 6"
-118,FALSE,2022-04-07 7:05:11,Are,Are,,NonPosting,,AVERAGE,,,,,,,,,Are,,,Are,AVERAGE,T,F,F,F,,,,F,F,,,Are,"1, 3, 4, 6"
-125,FALSE,2022-04-07 7:05:11,Over,Over,,NonPosting,,AVERAGE,,,,,,,,,Over,,,Over,AVERAGE,T,F,F,F,,,,F,F,,,Over,"1, 3, 4, 6"
-121,FALSE,2022-04-07 7:05:11,Nine,Nine,,COGS,,AVERAGE,,,,,,,,,Nine,,,Nine,AVERAGE,T,F,F,F,2021-02-10 9:19:54,,,F,F,,,Nine,"1, 3, 4, 6"
-117,FALSE,2022-04-07 7:05:11,Thousand,Thousand,,NonPosting,,AVERAGE,,,,,,,,,Thousand,,,Thousand,AVERAGE,T,F,F,F,,,,F,F,,,Thousand,"1, 3, 4, 6"
+id,_fivetran_deleted,_fivetran_synced,accountsearchdisplayname,accountsearchdisplaynamecopy,acctnumber,accttype,billableexpensesacct,cashflowrate,category1099misc,class,currency,custrecord_fam_account_showinfixedasset,date_deleted,deferralacct,department,description,displaynamewithhierarchy,eliminate,externalid,fullname,generalrate,includechildren,inventory,isinactive,issummary,lastmodifieddate,location,parent,reconcilewithmatching,revalue,sbankname,sbankroutingnumber,sspecacct,subsidiary,eliminate
+133,FALSE,2022-04-07 7:05:11,Power,Power,,OthExpense,,AVERAGE,,,,"3, 4, 5, 6",,,,,Power,,Allowance for Doubtful Account,Power,AVERAGE,T,F,F,F,2017-05-21 1:41:07,,,F,F,,,Power,"1, 3, 4, 6",F
+157,FALSE,2022-04-07 7:05:11,Levels,Levels,,OthExpense,,AVERAGE,,,,"3, 4, 5, 6",,,,,Levels,,,Levels,AVERAGE,T,F,F,F,2017-05-02 22:46:13,,,F,F,,,Levels,"1, 3, 4, 6",F
+118,FALSE,2022-04-07 7:05:11,Are,Are,,NonPosting,,AVERAGE,,,,,,,,,Are,,,Are,AVERAGE,T,F,F,F,,,,F,F,,,Are,"1, 3, 4, 6",F
+125,FALSE,2022-04-07 7:05:11,Over,Over,,NonPosting,,AVERAGE,,,,,,,,,Over,,,Over,AVERAGE,T,F,F,F,,,,F,F,,,Over,"1, 3, 4, 6",F
+121,FALSE,2022-04-07 7:05:11,Nine,Nine,,COGS,,AVERAGE,,,,,,,,,Nine,,,Nine,AVERAGE,T,F,F,F,2021-02-10 9:19:54,,,F,F,,,Nine,"1, 3, 4, 6",
+117,FALSE,2022-04-07 7:05:11,Thousand,Thousand,,NonPosting,,AVERAGE,,,,,,,,,Thousand,,,Thousand,AVERAGE,T,F,F,F,,,,F,F,,,Thousand,"1, 3, 4, 6",T

integration_tests/seeds/netsuite_accounts_data.csv

@@ -1,11 +1,11 @@
 account_id,parent_id,name,type_name,account_number,general_rate_type,is_leftside,is_balancesheet
-530,,8769f2a4710dce527c26a6962560eb68,eb45bd2725cb8ac50d0795432a918f86,10000,a2770969c827f0f2910f6179418462df,No,No
-531,530,4f776429f32750022c01847c813b00cd,eb45bd2725cb8ac50d0795432a918f86,10001,a2770969c827f0f2910f6179418462df,No,No
-532,530,e17c5b5a8046fa938ae6c28848445e5a,eb45bd2725cb8ac50d0795432a918f86,10002,a2770969c827f0f2910f6179418462df,No,No
-533,530,8deba561c94866c69412c5ad9785955d,eb45bd2725cb8ac50d0795432a918f86,10003,a2770969c827f0f2910f6179418462df,Yes,Yes
-534,530,4d17a0007c857ebc8fa6be1e6d8e6eb2,eb45bd2725cb8ac50d0795432a918f86,10004,a2770969c827f0f2910f6179418462df,Yes,Yes
-535,530,8bc09cee29171553028ae27eebf6cdf5,eb45bd2725cb8ac50d0795432a918f86,10005,a2770969c827f0f2910f6179418462df,Yes,Yes
-1037,530,7790e9be35e2c705c2c62b04b503d962,eb45bd2725cb8ac50d0795432a918f86,10006,a2770969c827f0f2910f6179418462df,No,No
-1044,530,d739f290e21e40f880af009b06f115d2,eb45bd2725cb8ac50d0795432a918f86,10007,a2770969c827f0f2910f6179418462df,No,No
-1067,530,29c0ed5ce3ffa0115971fe05cd92e577,eb45bd2725cb8ac50d0795432a918f86,10008,a2770969c827f0f2910f6179418462df,Yes,Yes
-1050,530,f3537e12e19781fd44ca2b77be86afc8,eb45bd2725cb8ac50d0795432a918f86,10021,a2770969c827f0f2910f6179418462df,No,No
+530,,8769f2a4710dce527c26a6962560eb68,eb45bd2725cb8ac50d0795432a918f86,10000,a2770969c827f0f2910f6179418462df,f,f
+531,530,4f776429f32750022c01847c813b00cd,eb45bd2725cb8ac50d0795432a918f86,10001,a2770969c827f0f2910f6179418462df,f,f
+532,530,e17c5b5a8046fa938ae6c28848445e5a,eb45bd2725cb8ac50d0795432a918f86,10002,a2770969c827f0f2910f6179418462df,f,f
+533,530,8deba561c94866c69412c5ad9785955d,eb45bd2725cb8ac50d0795432a918f86,10003,a2770969c827f0f2910f6179418462df,t,t
+534,530,4d17a0007c857ebc8fa6be1e6d8e6eb2,eb45bd2725cb8ac50d0795432a918f86,10004,a2770969c827f0f2910f6179418462df,t,t
+535,530,8bc09cee29171553028ae27eebf6cdf5,eb45bd2725cb8ac50d0795432a918f86,10005,a2770969c827f0f2910f6179418462df,t,t
+1037,530,7790e9be35e2c705c2c62b04b503d962,eb45bd2725cb8ac50d0795432a918f86,10006,a2770969c827f0f2910f6179418462df,f,f
+1044,530,d739f290e21e40f880af009b06f115d2,eb45bd2725cb8ac50d0795432a918f86,10007,a2770969c827f0f2910f6179418462df,f,f
+1067,530,29c0ed5ce3ffa0115971fe05cd92e577,eb45bd2725cb8ac50d0795432a918f86,10008,a2770969c827f0f2910f6179418462df,t,t
+1050,530,f3537e12e19781fd44ca2b77be86afc8,eb45bd2725cb8ac50d0795432a918f86,10021,a2770969c827f0f2910f6179418462df,f,f

models/netsuite/netsuite__balance_sheet.sql

@@ -62,12 +62,15 @@ balance_sheet as (
     {{ fivetran_utils.persist_pass_through_columns('accounts_pass_through_columns', identifier='accounts') }},
 
     case
-      when lower(accounts.is_balancesheet) = 'f' or lower(transactions_with_converted_amounts.account_category) = 'equity' then -converted_amount_using_transaction_accounting_period
-      when lower(accounts.is_leftside) = 'f' then -converted_amount_using_reporting_month
-      when lower(accounts.is_leftside) = 't' then converted_amount_using_reporting_month
+      when lower(accounts.is_balancesheet) = 'f' and lower(accounts.general_rate_type) in ('historical', 'average') then -converted_amount_using_transaction_accounting_period
+      when lower(accounts.is_balancesheet) = 'f' then -converted_amount_using_reporting_month
+      when lower(accounts.is_balancesheet) = 't' and lower(accounts.is_leftside) = 'f' and lower(accounts.general_rate_type) in ('historical', 'average') then -converted_amount_using_transaction_accounting_period
+      when lower(accounts.is_balancesheet) = 't' and lower(accounts.is_leftside) = 't' and lower(accounts.general_rate_type) in ('historical', 'average') then converted_amount_using_transaction_accounting_period
+      when lower(accounts.is_balancesheet) = 't' and lower(accounts.is_leftside) = 'f' then -converted_amount_using_reporting_month
+      when lower(accounts.is_balancesheet) = 't' and lower(accounts.is_leftside) = 't' then converted_amount_using_reporting_month
       else 0
         end as converted_amount,
-
+        
     case
       when lower(accounts.type_name) = 'bank' then 1
       when lower(accounts.type_name) = 'accounts receivable' then 2

models/netsuite2/netsuite2__balance_sheet.sql

@@ -72,12 +72,15 @@ balance_sheet as (
     {{ fivetran_utils.persist_pass_through_columns('accounts_pass_through_columns', identifier='accounts') }},
 
     case
-      when not accounts.is_balancesheet or lower(transactions_with_converted_amounts.account_category) = 'equity' then -converted_amount_using_transaction_accounting_period
-      when not accounts.is_leftside then -converted_amount_using_reporting_month
-      when accounts.is_leftside then converted_amount_using_reporting_month
+      when not accounts.is_balancesheet and lower(accounts.general_rate_type) in ('historical', 'average') then -converted_amount_using_transaction_accounting_period
+      when not accounts.is_balancesheet then -converted_amount_using_reporting_month
+      when accounts.is_balancesheet and not accounts.is_leftside and lower(accounts.general_rate_type) in ('historical', 'average') then -converted_amount_using_transaction_accounting_period
+      when accounts.is_balancesheet and accounts.is_leftside and lower(accounts.general_rate_type) in ('historical', 'average') then converted_amount_using_transaction_accounting_period
+      when accounts.is_balancesheet and not accounts.is_leftside then -converted_amount_using_reporting_month
+      when accounts.is_balancesheet and accounts.is_leftside then converted_amount_using_reporting_month
       else 0
         end as converted_amount,
-    
+
     case
       when lower(accounts.account_type_id) = 'bank' then 1
       when lower(accounts.account_type_id) = 'acctrec' then 2
@@ -154,7 +157,7 @@ balance_sheet as (
     {% endif %}
 
     case
-      when lower(account_category) = 'equity' or is_income_statement then converted_amount_using_transaction_accounting_period
+      when lower(accounts.general_rate_type) in ('historical', 'average') then converted_amount_using_transaction_accounting_period
       else converted_amount_using_reporting_month
         end as converted_amount,
     16 as balance_sheet_sort_helper

models/netsuite2/netsuite2__transaction_details.sql

@@ -111,7 +111,7 @@ transaction_details as (
     accounts.is_leftside as is_account_leftside,
     lower(accounts.account_type_id) = 'acctpay' as is_accounts_payable,
     lower(accounts.account_type_id) = 'acctrec' as is_accounts_receivable,
-    lower(accounts.name) like '%intercompany%' as is_account_intercompany,
+    accounts.is_eliminate as is_account_intercompany,
     coalesce(parent_account.name, accounts.name) as parent_account_name,
     lower(accounts.account_type_id) in ('expense', 'othexpense', 'deferexpense') as is_expense_account,
     lower(accounts.account_type_id) in ('income', 'othincome') as is_income_account,

packages.yml

@@ -1,3 +1,3 @@
 packages:
-- package: fivetran/netsuite_source
-  version: [">=0.7.0", "<0.8.0"]
+ - package: fivetran/netsuite_source
+   version: [">=0.8.0", "<0.9.0"]