Merge pull request #16 from fivetran/feature/union_source_data

Feature/union source data
fivetran · Nov 3, 2021 · 7633675 · 7633675
2 parents c262e5b + 9408a75
commit 7633675
Show file tree

Hide file tree

Showing 14 changed files with 113 additions and 38 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,15 @@
+# dbt_xero v0.3.0
+
+## Features
+- Allow for multiple sources by unioning source tables across multiple Xero connectors.
+([#11](https://github.com/fivetran/dbt_xero_source/pull/11))
+  - Refer to the unioning multiple Xero connectors section of the [README](https://github.com/fivetran/dbt_xero/tree/main#unioning-multiple-xero-connectors) for more details.
+
+## Under the Hood
+- Unioning: The unioning occurs in Xero source package using the `fivetran_utils.union_data` macro. ([#16](https://github.com/fivetran/dbt_xero/pull/16))
+- Unique tests: Because columns that were previously used for unique tests may now have duplicate fields across multiple sources, these columns are combined with the new `source_relation` column for unique tests and tested using the `dbt_utils.unique_combination_of_columns` macro. ([#16](https://github.com/fivetran/dbt_xero/pull/16))
+- Source Relation column: To distinguish which source each field comes from, we added a new `source_relation` column in each staging model and applied the `fivetran_utils.source_relation` macro. ([#16](https://github.com/fivetran/dbt_xero/pull/16))
+- Utils Materialization: We have made the default materialization of the `utils` folder to be ephemeral. ([#16](https://github.com/fivetran/dbt_xero/pull/16))
+
+# dbt_xero v0.1.0 -> v0.2.0
+Refer to the relevant release notes on the Github repository for specific details for the previous releases. Thank you!
diff --git a/README.md b/README.md
@@ -29,7 +29,7 @@ Include in your `packages.yml`
 ```yaml
 packages:
   - package: fivetran/xero
-    version: [">=0.2.0", "<0.3.0"]
+    version: [">=0.3.0", "<0.4.0"]
 ```
 
 ## Configuration
@@ -47,6 +47,19 @@ vars:
     xero_database: your_database_name 
 ```
 
+### Unioning Multiple Xero Connectors
+If you have multiple Xero connectors in Fivetran and would like to use this package on all of them simultaneously, we have provided functionality to do so. The package will union all of the data together and pass the unioned table into the transformations. You will be able to see which source it came from in the `source_relation` column of each model. To use this functionality, you will need to set **either** (**note that you cannot use both**) the `union_schemas` or `union_databases` variables:
+
+```yml
+# dbt_project.yml
+...
+config-version: 2
+vars:
+  xero_source:
+    union_schemas: ['xero_us','xero_ca'] # use this if the data is in different schemas/datasets of the same database/project
+    union_databases: ['xero_us','xero_ca'] # use this if the data is in different databases/projects but uses the same schema name
+```
+
 ### Disabling and Enabling Models
 
 When setting up your Xero connection in Fivetran, it is possible that not every table this package expects will be synced. This can occur because you either don't use that functionality in Xero or have actively decided to not sync some tables. In order to disable the relevant functionality in the package, you will need to add the relevant variables.

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'xero'
-version: '0.2.0'
+version: '0.3.0'
 config-version: 2
 require-dbt-version: [">=0.20.0"]
 
@@ -18,4 +18,6 @@ vars:
 models:
   xero:
     +materialized: table
-    +schema: xero
+    +schema: xero
+    utilities:
+       +materialized: ephemeral
diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'xero_integration_tests'
-version: '0.2.0'
+version: '0.3.0'
 profile: 'integration_tests'
 config-version: 2
 

diff --git a/models/xero.yml b/models/xero.yml
@@ -7,7 +7,7 @@ models:
       - name: profit_and_loss_id
         description: Unique identifier for each record in the profit and loss report model.
         tests:
-          - unique
+          - unique  
           - not_null
       - name: date_month
         description: The month of activity.
@@ -23,9 +23,19 @@ models:
         description: The class of account, e.g. ASSET, EQUITY, EXPENSE, LIABILITY, REVENUE
       - name: net_amount
         description: The net movement amount.
+      - name: source_relation
+        description: >
+          The source where this data was pulled from. If you are making use of the `union_schemas` variable, this will be the source schema.
+          If you are making use of the `union_databases` variable, this will be the source database. If you are not unioining together multiple
+          sources, this will be an empty string.
 
   - name: xero__invoice_line_items
     description: Each record represents an invoice line item, enriched with account, contact and invoice information.
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - line_item_id
+            - source_relation
     columns:
       - name: _fivetran_synced
         description: "{{ doc('_fivetran_synced') }}"
@@ -46,7 +56,6 @@ models:
       - name: line_item_id
         description: Xero identifier for the line item.
         tests:
-          - unique
           - not_null
       - name: quantity
         description: The quantity of the line item.
@@ -96,14 +105,22 @@ models:
         description: The class of account, e.g. ASSET, EQUITY, EXPENSE, LIABILITY, REVENUE
       - name: contact_name
         description: The name of the associated contact.
+      - name: source_relation
+        description: >
+          The source where this data was pulled from. If you are making use of the `union_schemas` variable, this will be the source schema.
+          If you are making use of the `union_databases` variable, this will be the source database. If you are not unioining together multiple
+          sources, this will be an empty string.
 
   - name: xero__general_ledger
     description: Each record represents a journal line item. This ledger is used to create the balance sheet and profit and loss.
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - journal_line_id
+            - source_relation
     columns:
       - name: journal_line_id
         description: Xero identifier for the journal line.
-        tests:
-          - unique
       - name: journal_id
         description: Xero identifier for the journal entry
         tests:
@@ -158,9 +175,20 @@ models:
         description: ID of the associated contact, where relevant.
       - name: contact_name
         description: Name of the associated contact, where relevant.
+      - name: source_relation
+        description: >
+          The source where this data was pulled from. If you are making use of the `union_schemas` variable, this will be the source schema.
+          If you are making use of the `union_databases` variable, this will be the source database. If you are not unioining together multiple
+          sources, this will be an empty string.
 
   - name: xero__balance_sheet_report
     description: Each record represents the state of the balance sheet for a given account on a given month.
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - source_relation
+            - date_month
+            - account_name
     columns:
       - name: date_month
         description: The month a given record is reporting on.
@@ -176,3 +204,8 @@ models:
         description: The type of the related account.
       - name: net_amount
         description: The net balance amount.
+      - name: source_relation
+        description: >
+          The source where this data was pulled from. If you are making use of the `union_schemas` variable, this will be the source schema.
+          If you are making use of the `union_databases` variable, this will be the source database. If you are not unioining together multiple
+          sources, this will be an empty string.
diff --git a/models/xero__balance_sheet_report.sql b/models/xero__balance_sheet_report.sql
@@ -48,12 +48,13 @@ with calendar as (
             when ledger.account_class in ('ASSET','EQUITY','LIABILITY') then ledger.account_class
             else 'EQUITY'
         end as account_class,
+        ledger.source_relation, 
         sum(ledger.net_amount) as net_amount
     from calendar
     inner join ledger
         on calendar.date_month >= cast({{ dbt_utils.date_trunc('month', 'ledger.journal_date') }} as date)
     cross join year_end
-    {{ dbt_utils.group_by(6) }}
+    {{ dbt_utils.group_by(7) }}
 
 )
 

diff --git a/models/xero__general_ledger.sql b/models/xero__general_ledger.sql
@@ -45,6 +45,7 @@ with journals as (
         journals.reference,
         journals.source_id,
         journals.source_type,
+        journals.source_relation, 
 
         journal_lines.journal_line_id,
         journal_lines.account_code,
@@ -69,9 +70,11 @@ with journals as (
 
     from journals
     left join journal_lines
-        on journals.journal_id = journal_lines.journal_id
+        on (journals.journal_id = journal_lines.journal_id
+        and journals.source_relation = journal_lines.source_relation)
     left join accounts
-        on accounts.account_id = journal_lines.account_id
+        on (accounts.account_id = journal_lines.account_id
+        and accounts.source_relation = journal_lines.source_relation)
 
 ), first_contact as (
 
@@ -87,14 +90,17 @@ with journals as (
 
         ) as contact_id
     from joined
-    left join invoices
-        using (invoice_id)
+    left join invoices 
+        on (joined.invoice_id = invoices.invoice_id
+        and joined.source_relation = invoices.source_relation)
     left join bank_transactions
-        using (bank_transaction_id)
+        on (joined.bank_transaction_id = bank_transactions.bank_transaction_id
+        and joined.source_relation = bank_transactions.source_relation)
 
     {% if var('xero__using_credit_note', True) %}
-    left join credit_notes
-        using (credit_note_id)
+    left join credit_notes 
+        on (joined.credit_note_id = credit_notes.credit_note_id
+        and joined.source_relation = credit_notes.source_relation)
     {% endif %}
 
 ), second_contact as (
@@ -104,7 +110,8 @@ with journals as (
         contacts.contact_name
     from first_contact
     left join contacts 
-        using (contact_id)
+        on (first_contact.contact_id = contacts.contact_id
+        and first_contact.source_relation = contacts.source_relation)
 
 )
 

diff --git a/models/xero__invoice_line_items.sql b/models/xero__invoice_line_items.sql
@@ -47,11 +47,14 @@ with line_items as (
 
     from line_items
     left join invoices
-        using (invoice_id)
+        on (line_items.invoice_id = invoices.invoice_id
+        and line_items.source_relation = invoices.source_relation)
     left join accounts
-        using (account_code)
+        on (line_items.account_code = accounts.account_code
+        and line_items.source_relation = accounts.source_relation)
     left join contacts
-        using (contact_id)
+        on (invoices.contact_id = contacts.contact_id
+        and invoices.source_relation = contacts.source_relation)
 
 )
 

diff --git a/models/xero__profit_and_loss_report.sql b/models/xero__profit_and_loss_report.sql
@@ -11,19 +11,20 @@ with calendar as (
 ), joined as (
 
     select 
-        {{ dbt_utils.surrogate_key(['calendar.date_month','ledger.account_id']) }} as profit_and_loss_id,
+        {{ dbt_utils.surrogate_key(['calendar.date_month','ledger.account_id','ledger.source_relation']) }} as profit_and_loss_id,
         calendar.date_month, 
         ledger.account_id,
         ledger.account_name,
         ledger.account_code,
         ledger.account_type, 
         ledger.account_class, 
+        ledger.source_relation, 
         coalesce(sum(ledger.net_amount * -1),0) as net_amount
     from calendar
     left join ledger
         on calendar.date_month = cast({{ dbt_utils.date_trunc('month', 'ledger.journal_date') }} as date)
     where ledger.account_class in ('REVENUE','EXPENSE')
-    {{ dbt_utils.group_by(7) }}
+    {{ dbt_utils.group_by(8) }}
 
 )
 

diff --git a/packages.yml b/packages.yml
@@ -1,3 +1,3 @@
 packages:
   - package: fivetran/xero_source
-    version: [">=0.2.0","<0.3.0"]
+    version: [">=0.3.0","<0.4.0"]