From 7721cc885f5b689eaf1a3fc43d22adf25adee352 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 23 Jul 2021 22:13:11 +0100 Subject: [PATCH 1/6] Updating package dependendcies --- packages.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages.yml b/packages.yml index 81f829b..86b2426 100644 --- a/packages.yml +++ b/packages.yml @@ -1,4 +1,4 @@ packages: - - package: fivetran/shopify_source - version: [">=0.4.0","<0.5.0"] \ No newline at end of file +- package: fivetran/shopify_source + version: [">=0.5.0", "<0.6.0"] From c42af553c9ebf51e5e2837997e1958bc56f9c3ec Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 23 Jul 2021 22:13:12 +0100 Subject: [PATCH 2/6] Updating require-dbt-version --- dbt_project.yml | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index b59be0b..54fbdd8 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,28 +1,23 @@ - name: 'shopify' -version: '0.4.2' +version: '0.5.0' config-version: 2 - -require-dbt-version: [">=0.18.0", "<0.20.0"] - -models: +require-dbt-version: [">=0.20.0"] +models: shopify: +schema: shopify +materialized: table intermediate: +materialized: ephemeral - vars: shopify: - shopify_customer: "{{ ref('stg_shopify__customer') }}" - shopify_order: "{{ ref('stg_shopify__order') }}" - shopify_order_line: "{{ ref('stg_shopify__order_line') }}" - shopify_order_line_refund: "{{ ref('stg_shopify__order_line_refund') }}" - shopify_product: "{{ ref('stg_shopify__product') }}" - shopify_product_variant: "{{ ref('stg_shopify__product_variant') }}" - shopify_transaction: "{{ ref('stg_shopify__transaction') }}" - shopify_refund: "{{ ref('stg_shopify__refund') }}" - shopify_order_adjustment: "{{ ref('stg_shopify__order_adjustment') }}" - + shopify_customer: "{{ ref('stg_shopify__customer') }}" + shopify_order: "{{ ref('stg_shopify__order') }}" + shopify_order_line: "{{ ref('stg_shopify__order_line') }}" + shopify_order_line_refund: "{{ ref('stg_shopify__order_line_refund') }}" + shopify_product: "{{ ref('stg_shopify__product') }}" + shopify_product_variant: "{{ ref('stg_shopify__product_variant') }}" + shopify_transaction: "{{ ref('stg_shopify__transaction') }}" + shopify_refund: "{{ ref('stg_shopify__refund') }}" + shopify_order_adjustment: "{{ ref('stg_shopify__order_adjustment') }}" dbt_utils_dispatch_list: [spark_utils, fivetran_utils] - fivetran_utils_dispatch_list: [spark_utils] \ No newline at end of file + fivetran_utils_dispatch_list: [spark_utils] From 51789bf641dfbf9387a1705a75e8a587b14d5d63 Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 23 Jul 2021 22:13:13 +0100 Subject: [PATCH 3/6] Updating dbt version in requirements.txt --- integration_tests/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index ca89302..b06ff52 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,3 +1,3 @@ -dbt==0.19.1 -dbt-spark==0.19.1 -dbt-spark[PyHive]==0.19.1 \ No newline at end of file +dbt==0.20.0 +dbt-spark==0.20.0 +dbt-spark[PyHive]==0.20.0 From 5249de087b76868737bf57a7245cbe65ca80d53f Mon Sep 17 00:00:00 2001 From: fivetran-joemarkiewicz Date: Tue, 27 Jul 2021 15:40:00 -0500 Subject: [PATCH 4/6] dbt 20 readme and minor edits --- README.md | 19 +++++++++++++++++++ dbt_project.yml | 2 -- integration_tests/dbt_project.yml | 6 +++++- packages.yml | 8 +++++--- 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9c32bee..0bd1a97 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +[![Apache License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) ![dbt logo and version](https://img.shields.io/static/v1?logo=dbt&label=dbt-version&message=0.20.x&color=orange) # Shopify This package models Shopify data from [Fivetran's connector](https://fivetran.com/docs/applications/shopify). It uses data in the format described by [this ERD](https://fivetran.com/docs/applications/shopify#schemainformation). @@ -21,6 +22,14 @@ This package contains transformation models, designed to work simultaneously wit ## Installation Instructions Check [dbt Hub](https://hub.getdbt.com/) for the latest installation instructions, or [read the dbt docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages. +Include in your `packages.yml` + +```yaml +packages: + - package: fivetran/shopify + version: [">=0.5.0", "<0.6.0"] +``` + ## Configuration By default, this package looks for your Shopify data in the `shopify` schema of your [target database](https://docs.getdbt.com/docs/running-a-dbt-project/using-the-command-line-interface/configure-your-profile). If this is not where your Shopify data is, add the following configuration to your `dbt_project.yml` file: @@ -74,6 +83,16 @@ on the best workflow for contributing to a package. ## Database support This package has been tested on BigQuery, Snowflake, Redshift, Postgres, and Databricks. +### Databricks Dispatch Configuration +dbt `v0.20.0` introduced a new project-level dispatch configuration that enables an "override" setting for all dispatched macros. If you are using a Databricks destination with this package you will need to add the below (or a variation of the below) dispatch configuration within your `dbt_project.yml`. This is required in order for the package to accurately search for macros within the `dbt-labs/spark_utils` then the `dbt-labs/dbt_utils` packages respectively. +```yml +# dbt_project.yml + +dispatch: + - macro_namespace: dbt_utils + search_order: ['spark_utils', 'dbt_utils'] +``` + ## Resources: - Provide [feedback](https://www.surveymonkey.com/r/DQ7K7WW) on our existing dbt packages or what you'd like to see next - Have questions, feedback, or need help? Book a time during our office hours [using Calendly](https://calendly.com/fivetran-solutions-team/fivetran-solutions-team-office-hours) or email us at solutions@fivetran.com diff --git a/dbt_project.yml b/dbt_project.yml index 54fbdd8..f0f59b5 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -19,5 +19,3 @@ vars: shopify_transaction: "{{ ref('stg_shopify__transaction') }}" shopify_refund: "{{ ref('stg_shopify__refund') }}" shopify_order_adjustment: "{{ ref('stg_shopify__order_adjustment') }}" - dbt_utils_dispatch_list: [spark_utils, fivetran_utils] - fivetran_utils_dispatch_list: [spark_utils] diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 6b16ff3..f68fe92 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'shopify_integration_tests' -version: '0.4.2' +version: '0.5.0' profile: 'integration_tests' config-version: 2 @@ -16,6 +16,10 @@ vars: refund_source: "{{ ref('shopify_refund_data') }}" transaction_source: "{{ ref('shopify_transaction_data') }}" +dispatch: + - macro_namespace: dbt_utils + search_order: ['spark_utils', 'dbt_utils'] + seeds: shopify_integration_tests: +quote_columns: "{{ true if target.type in ('redshift', 'postgres') else false }}" diff --git a/packages.yml b/packages.yml index 86b2426..f1ad427 100644 --- a/packages.yml +++ b/packages.yml @@ -1,4 +1,6 @@ packages: - -- package: fivetran/shopify_source - version: [">=0.5.0", "<0.6.0"] + # - package: fivetran/shopify_source + # version: [">=0.5.0", "<0.6.0"] + - git: https://github.com/fivetran/dbt_shopify_source.git + revision: feature/dbt20-compatibility + warn-unpinned: false \ No newline at end of file From 78d3e7a8c6d0f04d82aaef06b25da9bac4e52fe7 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Thu, 5 Aug 2021 09:17:33 -0500 Subject: [PATCH 5/6] Update integration_tests/requirements.txt --- integration_tests/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index b06ff52..dc77891 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,3 +1,3 @@ -dbt==0.20.0 -dbt-spark==0.20.0 -dbt-spark[PyHive]==0.20.0 +dbt==0.20.1rc1 +dbt-spark==0.20.1rc1 +dbt-spark[PyHive]==0.20.1rc1 From f67cdee218a04d89f7db1c4a14c83eaa5b266603 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Thu, 5 Aug 2021 10:13:39 -0500 Subject: [PATCH 6/6] Update packages.yml --- packages.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages.yml b/packages.yml index f1ad427..2a75941 100644 --- a/packages.yml +++ b/packages.yml @@ -1,6 +1,3 @@ packages: - # - package: fivetran/shopify_source - # version: [">=0.5.0", "<0.6.0"] - - git: https://github.com/fivetran/dbt_shopify_source.git - revision: feature/dbt20-compatibility - warn-unpinned: false \ No newline at end of file + - package: fivetran/shopify_source + version: [">=0.5.0", "<0.6.0"]