From 3f11435c790808548fa4bcf8ef95da3e0bb5e509 Mon Sep 17 00:00:00 2001 From: artc95 Date: Fri, 2 Feb 2024 17:43:31 +0100 Subject: [PATCH 01/41] switch dependency from dbt-sqlserver to dbt-fabric --- CHANGELOG.md | 5 +++++ dbt/adapters/synapse/__init__.py | 2 +- dbt/adapters/synapse/__version__.py | 2 +- dbt/adapters/synapse/synapse_adapter.py | 4 ++-- dbt/adapters/synapse/synapse_connection_manager.py | 4 ++-- dbt/adapters/synapse/synapse_credentials.py | 4 ++-- setup.py | 6 +++--- 7 files changed, 16 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 202eee12..9781eeaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ # Changelog +## v1.7.0 + +#### Under the hood +* Switch dependency from dbt-sqlserver to dbt-fabric + ## v1.4.0 #### Features diff --git a/dbt/adapters/synapse/__init__.py b/dbt/adapters/synapse/__init__.py index 9f2548ac..6a0d96fd 100644 --- a/dbt/adapters/synapse/__init__.py +++ b/dbt/adapters/synapse/__init__.py @@ -9,7 +9,7 @@ adapter=SynapseAdapter, credentials=SynapseCredentials, include_path=synapse.PACKAGE_PATH, - dependencies=["sqlserver"], + dependencies=["fabric"], ) __all__ = ["Plugin", "SynapseConnectionManager", "SynapseAdapter", "SynapseCredentials"] diff --git a/dbt/adapters/synapse/__version__.py b/dbt/adapters/synapse/__version__.py index d619c757..a55413d1 100644 --- a/dbt/adapters/synapse/__version__.py +++ b/dbt/adapters/synapse/__version__.py @@ -1 +1 @@ -version = "1.4.0" +version = "1.7.0" diff --git a/dbt/adapters/synapse/synapse_adapter.py b/dbt/adapters/synapse/synapse_adapter.py index a1608e3f..fcccde7b 100644 --- a/dbt/adapters/synapse/synapse_adapter.py +++ b/dbt/adapters/synapse/synapse_adapter.py @@ -1,7 +1,7 @@ -from dbt.adapters.sqlserver import SQLServerAdapter +from dbt.adapters.fabric import FabricAdapter from dbt.adapters.synapse.synapse_connection_manager import SynapseConnectionManager -class SynapseAdapter(SQLServerAdapter): +class SynapseAdapter(FabricAdapter): ConnectionManager = SynapseConnectionManager diff --git a/dbt/adapters/synapse/synapse_connection_manager.py b/dbt/adapters/synapse/synapse_connection_manager.py index d4d0a1d6..23103d1b 100644 --- a/dbt/adapters/synapse/synapse_connection_manager.py +++ b/dbt/adapters/synapse/synapse_connection_manager.py @@ -1,6 +1,6 @@ -from dbt.adapters.sqlserver import SQLServerConnectionManager +from dbt.adapters.fabric import FabricConnectionManager -class SynapseConnectionManager(SQLServerConnectionManager): +class SynapseConnectionManager(FabricConnectionManager): TYPE = "synapse" TOKEN = None diff --git a/dbt/adapters/synapse/synapse_credentials.py b/dbt/adapters/synapse/synapse_credentials.py index c17ea579..43f57a12 100644 --- a/dbt/adapters/synapse/synapse_credentials.py +++ b/dbt/adapters/synapse/synapse_credentials.py @@ -1,10 +1,10 @@ from dataclasses import dataclass -from dbt.adapters.sqlserver import SQLServerCredentials +from dbt.adapters.fabric import FabricCredentials @dataclass -class SynapseCredentials(SQLServerCredentials): +class SynapseCredentials(FabricCredentials): @property def type(self): return "synapse" diff --git a/setup.py b/setup.py index b82646ad..9a756d36 100644 --- a/setup.py +++ b/setup.py @@ -15,8 +15,8 @@ "Anders Swanson", "Sam Debruyn", ] -dbt_version = "1.4" -dbt_sqlserver_requirement = "dbt-sqlserver~=1.4.0" +dbt_version = "1.7" +dbt_fabric_requirement = "dbt-fabric~=1.7.0" description = """An Azure Synapse adapter plugin for dbt""" this_directory = os.path.abspath(os.path.dirname(__file__)) @@ -73,7 +73,7 @@ def run(self): url="https://github.com/dbt-msft/dbt-synapse", packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, - install_requires=[dbt_sqlserver_requirement], + install_requires=[dbt_fabric_requirement], cmdclass={ "verify": VerifyVersionCommand, }, From cd3b6afca49f8d7727aee95728c17068aa13c2a5 Mon Sep 17 00:00:00 2001 From: artc95 Date: Mon, 5 Feb 2024 10:13:55 +0100 Subject: [PATCH 02/41] fix indexes.sql for "USE statement not supported" error - switch dependency --- .../synapse/macros/adapters/indexes.sql | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/dbt/include/synapse/macros/adapters/indexes.sql b/dbt/include/synapse/macros/adapters/indexes.sql index 27cdb8b7..a1af97a9 100644 --- a/dbt/include/synapse/macros/adapters/indexes.sql +++ b/dbt/include/synapse/macros/adapters/indexes.sql @@ -126,3 +126,35 @@ declare @drop_remaining_indexes_last nvarchar(max) = ( {% macro create_nonclustered_index(columns, includes=False) %} {{ return(create_nonclustered_index(columns, includes=False)) }} {% endmacro %} + + +{% macro drop_fk_indexes_on_table(relation) -%} + {% call statement('find_references', fetch_result=true) %} + SELECT obj.name AS FK_NAME, + sch.name AS [schema_name], + tab1.name AS [table], + col1.name AS [column], + tab2.name AS [referenced_table], + col2.name AS [referenced_column] + FROM sys.foreign_key_columns fkc + INNER JOIN sys.objects obj + ON obj.object_id = fkc.constraint_object_id + INNER JOIN sys.tables tab1 + ON tab1.object_id = fkc.parent_object_id + INNER JOIN sys.schemas sch + ON tab1.schema_id = sch.schema_id + INNER JOIN sys.columns col1 + ON col1.column_id = parent_column_id AND col1.object_id = tab1.object_id + INNER JOIN sys.tables tab2 + ON tab2.object_id = fkc.referenced_object_id + INNER JOIN sys.columns col2 + ON col2.column_id = referenced_column_id AND col2.object_id = tab2.object_id + WHERE sch.name = '{{ relation.schema }}' and tab2.name = '{{ relation.identifier }}' + {% endcall %} + {% set references = load_result('find_references')['data'] %} + {% for reference in references -%} + {% call statement('main') -%} + alter table [{{reference[1]}}].[{{reference[2]}}] drop constraint [{{reference[0]}}] + {%- endcall %} + {% endfor %} +{% endmacro %} From 26e284961da42e779bcbd127278482ee77997c0f Mon Sep 17 00:00:00 2001 From: artc95 Date: Mon, 5 Feb 2024 11:01:17 +0100 Subject: [PATCH 03/41] add snapshot.sql for "Distribution option must be explicitly specified in 'CREATE TABLE AS SELECT'..." error - switch dependency --- .../materializations/snapshots/snapshot.sql | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 dbt/include/synapse/macros/materializations/snapshots/snapshot.sql diff --git a/dbt/include/synapse/macros/materializations/snapshots/snapshot.sql b/dbt/include/synapse/macros/materializations/snapshots/snapshot.sql new file mode 100644 index 00000000..effe87c3 --- /dev/null +++ b/dbt/include/synapse/macros/materializations/snapshots/snapshot.sql @@ -0,0 +1,61 @@ +{% macro synapse__create_columns(relation, columns) %} + {# default__ macro uses "add column" + TSQL preferes just "add" + #} + + {% set columns %} + {% for column in columns %} + , CAST(NULL AS {{column.data_type}}) AS {{column_name}} + {% endfor %} + {% endset %} + + {% set tempTableName %} + [{{relation.database}}].[{{ relation.schema }}].[{{ relation.identifier }}_{{ range(1300, 19000) | random }}] + {% endset %} + + {%- set index = config.get('index', default="CLUSTERED COLUMNSTORE INDEX") -%} + {%- set dist = config.get('dist', default="ROUND_ROBIN") -%} + {% set tempTable %} + CREATE TABLE {{tempTableName}} + WITH( + DISTRIBUTION = {{dist}}, + {{index}} + ) + AS SELECT * {{columns}} FROM [{{relation.database}}].[{{ relation.schema }}].[{{ relation.identifier }}] {{ information_schema_hints() }} + {% endset %} + + {% call statement('create_temp_table') -%} + {{ tempTable }} + {%- endcall %} + + {% set dropTable %} + DROP TABLE [{{relation.database}}].[{{ relation.schema }}].[{{ relation.identifier }}] + {% endset %} + + {% call statement('drop_table') -%} + {{ dropTable }} + {%- endcall %} + + {%- set index = config.get('index', default="CLUSTERED COLUMNSTORE INDEX") -%} + {%- set dist = config.get('dist', default="ROUND_ROBIN") -%} + {% set createTable %} + CREATE TABLE {{ relation }} + WITH( + DISTRIBUTION = {{dist}}, + {{index}} + ) + AS SELECT * FROM {{tempTableName}} {{ information_schema_hints() }} + {% endset %} + + {% call statement('create_Table') -%} + {{ createTable }} + {%- endcall %} + + {% set dropTempTable %} + DROP TABLE {{tempTableName}} + {% endset %} + + {% call statement('drop_temp_table') -%} + {{ dropTempTable }} + {%- endcall %} +{% endmacro %} From 33ce51474ae337abba95bcbac5bd1740c3cc47d1 Mon Sep 17 00:00:00 2001 From: artc95 Date: Mon, 5 Feb 2024 11:24:04 +0100 Subject: [PATCH 04/41] add "drop_schema" macro in schema.sql, without Synapse-unsupported "IF EXISTS" - switch dependency --- .../synapse/macros/adapters/schema.sql | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/dbt/include/synapse/macros/adapters/schema.sql b/dbt/include/synapse/macros/adapters/schema.sql index d24dd20e..fa472091 100644 --- a/dbt/include/synapse/macros/adapters/schema.sql +++ b/dbt/include/synapse/macros/adapters/schema.sql @@ -6,3 +6,23 @@ END {% endcall %} {% endmacro %} + +{% macro synapse__drop_schema(relation) -%} + {%- set relations_in_schema = list_relations_without_caching(relation) %} + + {% for row in relations_in_schema %} + {%- set schema_relation = api.Relation.create(database=relation.database, + schema=relation.schema, + identifier=row[1], + type=row[3] + ) -%} + {% do drop_relation(schema_relation) %} + {%- endfor %} + + {% call statement('drop_schema') -%} + IF EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ relation.without_identifier().schema }}') + BEGIN + EXEC('DROP SCHEMA [{{ relation.without_identifier().schema }}]') + END + {% endcall %} +{% endmacro %} From 3bc4d979ab944641aef9bafb816eb4c076266eb7 Mon Sep 17 00:00:00 2001 From: artc95 Date: Mon, 5 Feb 2024 12:37:49 +0100 Subject: [PATCH 05/41] skip ephemeral and changing_relation_type tests; change time(stamp) column type to datetime2 --- tests/functional/adapter/test_basic.py | 1 + tests/functional/adapter/test_changing_relation_type.py | 2 ++ tests/functional/adapter/test_data_types.py | 2 +- tests/functional/adapter/test_docs.py | 4 ++-- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py index d676a2ec..87133424 100644 --- a/tests/functional/adapter/test_basic.py +++ b/tests/functional/adapter/test_basic.py @@ -32,6 +32,7 @@ class TestEmptySynapse(BaseEmpty): pass +@pytest.mark.skip(reason="ephemeral not supported") class TestEphemeralSynapse(BaseEphemeral): pass diff --git a/tests/functional/adapter/test_changing_relation_type.py b/tests/functional/adapter/test_changing_relation_type.py index e80da716..e56f7f44 100644 --- a/tests/functional/adapter/test_changing_relation_type.py +++ b/tests/functional/adapter/test_changing_relation_type.py @@ -1,5 +1,7 @@ +import pytest from dbt.tests.adapter.relations.test_changing_relation_type import BaseChangeRelationTypeValidator +@pytest.mark.skip(reason="CTAS is not supported without a underlying table definition.") class TestChangeRelationTypesSynapse(BaseChangeRelationTypeValidator): pass diff --git a/tests/functional/adapter/test_data_types.py b/tests/functional/adapter/test_data_types.py index 7181ab31..b65f735b 100644 --- a/tests/functional/adapter/test_data_types.py +++ b/tests/functional/adapter/test_data_types.py @@ -46,7 +46,7 @@ def seeds(self): - name: expected config: column_types: - timestamp_col: "datetimeoffset" + timestamp_col: "datetime2" """ return { diff --git a/tests/functional/adapter/test_docs.py b/tests/functional/adapter/test_docs.py index e8fc4296..7ea26be4 100644 --- a/tests/functional/adapter/test_docs.py +++ b/tests/functional/adapter/test_docs.py @@ -22,7 +22,7 @@ def expected_catalog(self, project): role="dbo", id_type="int", text_type="varchar", - time_type="datetime", + time_type="datetime2", view_type="VIEW", table_type="BASE TABLE", model_stats=no_stats(), @@ -37,7 +37,7 @@ def expected_catalog(self, project): role="dbo", id_type="int", text_type="varchar", - time_type="datetime", + time_type="datetime2", bigint_type="int", view_type="VIEW", table_type="BASE TABLE", From cf7b5cffae2e64c1ea999f36f0326c7dcd5f8b0d Mon Sep 17 00:00:00 2001 From: artc95 Date: Tue, 6 Feb 2024 12:25:51 +0100 Subject: [PATCH 06/41] update "rename_relation" macro in relation.sql, fix "new name already in use...duplicate...not permitted" - switch dependency --- dbt/include/synapse/macros/adapters/relation.sql | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/dbt/include/synapse/macros/adapters/relation.sql b/dbt/include/synapse/macros/adapters/relation.sql index 7f3c6a94..be43be08 100644 --- a/dbt/include/synapse/macros/adapters/relation.sql +++ b/dbt/include/synapse/macros/adapters/relation.sql @@ -20,7 +20,17 @@ {% macro synapse__rename_relation(from_relation, to_relation) -%} {% call statement('rename_relation') -%} + -- drop all object types with to_relation.identifier name, to avoid error "new name already in use...duplicate...not permitted" + if object_id ('{{ to_relation.include(database=False) }}','V') is not null + begin + drop view {{ to_relation.include(database=False) }} + end + + if object_id ('{{ to_relation.include(database=False) }}','U') is not null + begin + drop table {{ to_relation.include(database=False) }} + end - rename object {{ from_relation.include(database=False) }} to {{ to_relation.identifier }} + rename object {{ from_relation.include(database=False) }} to {{ to_relation.identifier }} {%- endcall %} {% endmacro %} From 946a521aaedc986fec98632320b720350a030797 Mon Sep 17 00:00:00 2001 From: artc95 Date: Tue, 6 Feb 2024 12:32:31 +0100 Subject: [PATCH 07/41] do not skip changing_relation_type test, succeeds after "rename_relation" macro in relation.sql updated - switch dependency --- tests/functional/adapter/test_changing_relation_type.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/functional/adapter/test_changing_relation_type.py b/tests/functional/adapter/test_changing_relation_type.py index e56f7f44..e80da716 100644 --- a/tests/functional/adapter/test_changing_relation_type.py +++ b/tests/functional/adapter/test_changing_relation_type.py @@ -1,7 +1,5 @@ -import pytest from dbt.tests.adapter.relations.test_changing_relation_type import BaseChangeRelationTypeValidator -@pytest.mark.skip(reason="CTAS is not supported without a underlying table definition.") class TestChangeRelationTypesSynapse(BaseChangeRelationTypeValidator): pass From 40e1737c879b31e60e112071fc4fa4ddebe67bd0 Mon Sep 17 00:00:00 2001 From: artc95 Date: Tue, 6 Feb 2024 14:13:42 +0100 Subject: [PATCH 08/41] add truncate_relation macro in relation.sql, to overwrite fabric's macro that does not specify Distribution option in "CREATE TABLE AS SELECT" - switch dependency --- dbt/include/synapse/macros/adapters/relation.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dbt/include/synapse/macros/adapters/relation.sql b/dbt/include/synapse/macros/adapters/relation.sql index be43be08..8c1d3582 100644 --- a/dbt/include/synapse/macros/adapters/relation.sql +++ b/dbt/include/synapse/macros/adapters/relation.sql @@ -34,3 +34,9 @@ rename object {{ from_relation.include(database=False) }} to {{ to_relation.identifier }} {%- endcall %} {% endmacro %} + +{% macro synapse__truncate_relation(relation) %} + {% call statement('truncate_relation') -%} + truncate table {{ relation }} + {%- endcall %} +{% endmacro %} From eed57dc3e81a10797bbdd8d52ccb21ae53d006fe Mon Sep 17 00:00:00 2001 From: artc95 Date: Tue, 6 Feb 2024 15:07:03 +0100 Subject: [PATCH 09/41] overwrite assertion of "grant " in log_output to fix test_grants; link to dependency switch discussion in CHANGELOG.md - switch dependency --- CHANGELOG.md | 2 +- tests/functional/adapter/test_grants.py | 45 +++++++++++++++++++------ 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9781eeaa..b4abdbad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## v1.7.0 #### Under the hood -* Switch dependency from dbt-sqlserver to dbt-fabric +* Switch dependency from dbt-sqlserver to dbt-fabric (per https://github.com/dbt-msft/dbt-sqlserver/issues/441) ## v1.4.0 diff --git a/tests/functional/adapter/test_grants.py b/tests/functional/adapter/test_grants.py index 1eb193e0..026512d0 100644 --- a/tests/functional/adapter/test_grants.py +++ b/tests/functional/adapter/test_grants.py @@ -1,9 +1,12 @@ -import pytest from dbt.tests.adapter.grants.test_incremental_grants import BaseIncrementalGrants from dbt.tests.adapter.grants.test_invalid_grants import BaseInvalidGrants from dbt.tests.adapter.grants.test_model_grants import BaseModelGrants from dbt.tests.adapter.grants.test_seed_grants import BaseSeedGrants -from dbt.tests.adapter.grants.test_snapshot_grants import BaseSnapshotGrants +from dbt.tests.adapter.grants.test_snapshot_grants import ( + BaseSnapshotGrants, + user2_snapshot_schema_yml, +) +from dbt.tests.util import get_manifest, run_dbt, run_dbt_and_capture, write_file class TestIncrementalGrantsSynapse(BaseIncrementalGrants): @@ -27,11 +30,33 @@ class TestSeedGrantsSynapse(BaseSeedGrants): class TestSnapshotGrantsSynapse(BaseSnapshotGrants): - @pytest.fixture(scope="class") - def project_config_update(self): - return { - # ('42000', '[42000] [Microsoft][ODBC Driver 17 for SQL Server][SQL Server] - # Merge statements with a WHEN NOT MATCHED [BY TARGET] clause must - # target a hash distributed table. (100087) (SQLExecDirectW)') - "snapshots": {"test": {"dist": "HASH(id)", "index": "HEAP"}}, - } + def test_snapshot_grants(self, project, get_test_users): + test_users = get_test_users + select_privilege_name = self.privilege_grantee_name_overrides()["select"] + + # run the snapshot + results = run_dbt(["snapshot"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + snapshot_id = "snapshot.test.my_snapshot" + snapshot = manifest.nodes[snapshot_id] + expected = {select_privilege_name: [test_users[0]]} + assert snapshot.config.grants == expected + self.assert_expected_grants_match_actual(project, "my_snapshot", expected) + + # run it again, nothing should have changed + # since dbt selects into temporary table, drops existing, selects into original table name, + # SELECT needs to be granted again, so "grant " expected in log_output! + (results, log_output) = run_dbt_and_capture(["--debug", "snapshot"]) + assert len(results) == 1 + assert "revoke " not in log_output + assert "grant " in log_output # grant expected + self.assert_expected_grants_match_actual(project, "my_snapshot", expected) + + # change the grantee, assert it updates + updated_yaml = self.interpolate_name_overrides(user2_snapshot_schema_yml) + write_file(updated_yaml, project.project_root, "snapshots", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "snapshot"]) + assert len(results) == 1 + expected = {select_privilege_name: [test_users[1]]} + self.assert_expected_grants_match_actual(project, "my_snapshot", expected) From b644ed8bec46b5c82b7e97b3e9c4d6cee23dbb02 Mon Sep 17 00:00:00 2001 From: artc95 Date: Tue, 6 Feb 2024 15:24:22 +0100 Subject: [PATCH 10/41] update test_timestamps.py to expect datetime2 - switch dependency --- tests/functional/adapter/test_timestamps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/adapter/test_timestamps.py b/tests/functional/adapter/test_timestamps.py index 91c72f18..106e7905 100644 --- a/tests/functional/adapter/test_timestamps.py +++ b/tests/functional/adapter/test_timestamps.py @@ -15,4 +15,4 @@ def expected_schema(self): @pytest.fixture(scope="class") def expected_sql(self): - return '''select SYSDATETIME() as "current_timestamp"''' + return '''select CAST( SYSDATETIME() AS DATETIME2(6) ) as "current_timestamp"''' From 049a799b46a71aedcff57cc3994b29021720081c Mon Sep 17 00:00:00 2001 From: nszoni Date: Wed, 7 Feb 2024 16:07:15 +0100 Subject: [PATCH 11/41] fix versioning and reference to adhere to 1.4 --- CHANGELOG.md | 2 +- dbt/adapters/synapse/__version__.py | 2 +- setup.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4abdbad..7e3773db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ # Changelog -## v1.7.0 +## v1.4.1rc1 #### Under the hood * Switch dependency from dbt-sqlserver to dbt-fabric (per https://github.com/dbt-msft/dbt-sqlserver/issues/441) diff --git a/dbt/adapters/synapse/__version__.py b/dbt/adapters/synapse/__version__.py index a55413d1..9b363e24 100644 --- a/dbt/adapters/synapse/__version__.py +++ b/dbt/adapters/synapse/__version__.py @@ -1 +1 @@ -version = "1.7.0" +version = "1.4.1rc1" diff --git a/setup.py b/setup.py index 9a756d36..7cf2fe88 100644 --- a/setup.py +++ b/setup.py @@ -15,8 +15,8 @@ "Anders Swanson", "Sam Debruyn", ] -dbt_version = "1.7" -dbt_fabric_requirement = "dbt-fabric~=1.7.0" +dbt_version = "1.4.1rc1" +dbt_fabric_requirement = "dbt-fabric~=1.4.0rc3" description = """An Azure Synapse adapter plugin for dbt""" this_directory = os.path.abspath(os.path.dirname(__file__)) From 9185ef602a0e0280bf3bdf9da6a908b838b27918 Mon Sep 17 00:00:00 2001 From: nszoni Date: Wed, 7 Feb 2024 16:09:34 +0100 Subject: [PATCH 12/41] add pyodbc binary override for ARM --- dev_requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index b4833c9b..02e93bd0 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -4,7 +4,8 @@ wheel==0.40.0 pre-commit==2.21.0;python_version<"3.8" pre-commit==3.3.1;python_version>="3.8" pytest-dotenv==0.5.2 -dbt-tests-adapter~=1.4.5 +dbt-tests-adapter~=1.4.9 +pyodbc==4.0.39 --no-binary :all: aiohttp==3.8.3 azure-mgmt-synapse==2.0.0 flaky==3.7.0 From 7b356ebe73af311c50546782130f679d72c4b92b Mon Sep 17 00:00:00 2001 From: nszoni Date: Wed, 7 Feb 2024 16:26:16 +0100 Subject: [PATCH 13/41] prep for 1.5 rc --- CHANGELOG.md | 4 ++++ dbt/adapters/synapse/__version__.py | 2 +- dev_requirements.txt | 2 +- setup.py | 4 ++-- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e3773db..f47ea109 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ # Changelog +## v.1.5.0rc1 + +* Support for [dbt-core 1.5](https://github.com/dbt-labs/dbt-core/releases/tag/v1.5.0) + ## v1.4.1rc1 #### Under the hood diff --git a/dbt/adapters/synapse/__version__.py b/dbt/adapters/synapse/__version__.py index 9b363e24..fa6c5a1a 100644 --- a/dbt/adapters/synapse/__version__.py +++ b/dbt/adapters/synapse/__version__.py @@ -1 +1 @@ -version = "1.4.1rc1" +version = "1.5.0rc1" diff --git a/dev_requirements.txt b/dev_requirements.txt index 02e93bd0..802f4848 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -4,7 +4,7 @@ wheel==0.40.0 pre-commit==2.21.0;python_version<"3.8" pre-commit==3.3.1;python_version>="3.8" pytest-dotenv==0.5.2 -dbt-tests-adapter~=1.4.9 +dbt-tests-adapter~=1.5.9 pyodbc==4.0.39 --no-binary :all: aiohttp==3.8.3 azure-mgmt-synapse==2.0.0 diff --git a/setup.py b/setup.py index 7cf2fe88..25b6d2f8 100644 --- a/setup.py +++ b/setup.py @@ -15,8 +15,8 @@ "Anders Swanson", "Sam Debruyn", ] -dbt_version = "1.4.1rc1" -dbt_fabric_requirement = "dbt-fabric~=1.4.0rc3" +dbt_version = "1.5.0rc1" +dbt_fabric_requirement = "dbt-fabric~=1.5.0" description = """An Azure Synapse adapter plugin for dbt""" this_directory = os.path.abspath(os.path.dirname(__file__)) From ede60e4033076db8a9ed3604d529803c5dc57097 Mon Sep 17 00:00:00 2001 From: artc95 Date: Wed, 7 Feb 2024 16:55:39 +0100 Subject: [PATCH 14/41] overwrite "create_schema" method in SynapseAdapter, to use added macro "synapse__create_schema_with_authorization", fixed failing test_schema.py by adding "schema_authorization" variable to dbt_profile --- dbt/adapters/synapse/synapse_adapter.py | 20 +++++++++++++ .../synapse/macros/adapters/schema.sql | 9 ++++++ tests/functional/adapter/test_schema.py | 29 +++++++++++++++---- 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/dbt/adapters/synapse/synapse_adapter.py b/dbt/adapters/synapse/synapse_adapter.py index fcccde7b..5c02b579 100644 --- a/dbt/adapters/synapse/synapse_adapter.py +++ b/dbt/adapters/synapse/synapse_adapter.py @@ -1,7 +1,27 @@ +from dbt.adapters.base.relation import BaseRelation +from dbt.adapters.cache import _make_ref_key_dict from dbt.adapters.fabric import FabricAdapter +from dbt.adapters.sql.impl import CREATE_SCHEMA_MACRO_NAME +from dbt.events.functions import fire_event +from dbt.events.types import SchemaCreation from dbt.adapters.synapse.synapse_connection_manager import SynapseConnectionManager class SynapseAdapter(FabricAdapter): ConnectionManager = SynapseConnectionManager + + def create_schema(self, relation: BaseRelation) -> None: + relation = relation.without_identifier() + fire_event(SchemaCreation(relation=_make_ref_key_dict(relation))) + macro_name = CREATE_SCHEMA_MACRO_NAME + kwargs = { + "relation": relation, + } + + if self.config.credentials.schema_authorization: + kwargs["schema_authorization"] = self.config.credentials.schema_authorization + macro_name = "synapse__create_schema_with_authorization" + + self.execute_macro(macro_name, kwargs=kwargs) + self.commit_if_has_connection() diff --git a/dbt/include/synapse/macros/adapters/schema.sql b/dbt/include/synapse/macros/adapters/schema.sql index fa472091..106496fb 100644 --- a/dbt/include/synapse/macros/adapters/schema.sql +++ b/dbt/include/synapse/macros/adapters/schema.sql @@ -7,6 +7,15 @@ {% endcall %} {% endmacro %} +{% macro synapse__create_schema_with_authorization(relation, schema_authorization) -%} + {% call statement('create_schema') -%} + IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ relation.schema }}') + BEGIN + EXEC('CREATE SCHEMA [{{ relation.schema }}] AUTHORIZATION [{{ schema_authorization }}]') + END + {% endcall %} +{% endmacro %} + {% macro synapse__drop_schema(relation) -%} {%- set relations_in_schema = list_relations_without_caching(relation) %} diff --git a/tests/functional/adapter/test_schema.py b/tests/functional/adapter/test_schema.py index 90570610..c283555d 100644 --- a/tests/functional/adapter/test_schema.py +++ b/tests/functional/adapter/test_schema.py @@ -1,9 +1,33 @@ import os import pytest +from conftest import _profile_ci_azure_auto, _profile_user, _profile_user_azure from dbt.tests.util import run_dbt +@pytest.fixture(scope="class") +def dbt_profile_target(request): + profile = request.config.getoption("--profile") + + if profile == "ci_azure_auto": + return { + **_profile_ci_azure_auto(), + **{"schema_authorization": "{{ env_var('DBT_TEST_USER_1') }}"}, + } + if profile == "user": + return { + **_profile_user(), + **{"schema_authorization": "{{ env_var('DBT_TEST_USER_1') }}"}, + } + if profile == "user_azure": + return { + **_profile_user_azure(), + **{"schema_authorization": "{{ env_var('DBT_TEST_USER_1') }}"}, + } + + raise ValueError(f"Unknown profile: {profile}") + + class TestSchemaCreation: @pytest.fixture(scope="class") def models(self): @@ -14,11 +38,6 @@ def models(self): """, } - @staticmethod - @pytest.fixture(scope="class") - def dbt_profile_target_update(): - return {"schema_authorization": "{{ env_var('DBT_TEST_USER_1') }}"} - @staticmethod def _verify_schema_owner(schema_name, owner, project): get_schema_owner = f""" From 34adcc5556fefb8b93c9c881fae2fa533041d709 Mon Sep 17 00:00:00 2001 From: artc95 Date: Wed, 7 Feb 2024 17:45:10 +0100 Subject: [PATCH 15/41] adjust dbt_version in setup.py to 1.4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7cf2fe88..dec7e0c8 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ "Anders Swanson", "Sam Debruyn", ] -dbt_version = "1.4.1rc1" +dbt_version = "1.4" dbt_fabric_requirement = "dbt-fabric~=1.4.0rc3" description = """An Azure Synapse adapter plugin for dbt""" From b115a23870db8778637080a68da999a1aa3797cf Mon Sep 17 00:00:00 2001 From: nszoni Date: Fri, 9 Feb 2024 17:35:08 +0100 Subject: [PATCH 16/41] add constraint ddl macro and extend materialization --- .../models/table/create_table_as.sql | 39 ++++++++++++++----- .../models/table/create_table_constraints.sql | 17 ++++++++ .../models/view/create_view_as.sql | 15 ++++++- 3 files changed, 59 insertions(+), 12 deletions(-) create mode 100644 dbt/include/synapse/macros/materializations/models/table/create_table_constraints.sql diff --git a/dbt/include/synapse/macros/materializations/models/table/create_table_as.sql b/dbt/include/synapse/macros/materializations/models/table/create_table_as.sql index 60331e50..744e7872 100644 --- a/dbt/include/synapse/macros/materializations/models/table/create_table_as.sql +++ b/dbt/include/synapse/macros/materializations/models/table/create_table_as.sql @@ -10,16 +10,35 @@ {{ synapse__drop_relation_script(relation) }} - EXEC('create view [{{ tmp_relation.schema }}].[{{ tmp_relation.identifier }}] as - {{ temp_view_sql }} - '); - - CREATE TABLE {{ relation.include(database=False) }} - WITH( - DISTRIBUTION = {{dist}}, - {{index}} - ) - AS (SELECT * FROM [{{ tmp_relation.schema }}].[{{ tmp_relation.identifier }}]) + {{ synapse__create_view_as(tmp_relation, sql) }} + + {% set contract_config = config.get('contract') %} + + {% if contract_config.enforced %} + + {{exceptions.warn("Model contracts cannot be enforced by !")}} + + CREATE TABLE [{{relation.schema}}].[{{relation.identifier}}] + {{ synapse__build_columns_constraints(tmp_relation) }} + WITH( + DISTRIBUTION = {{dist}}, + {{index}} + ) + {{ get_assert_columns_equivalent(sql) }} + + {% set listColumns %} + {% for column in model['columns'] %} + {{ "["~column~"]" }}{{ ", " if not loop.last }} + {% endfor %} + {%endset%} + {{ synapse__build_model_constraints(relation) }} + + INSERT INTO [{{relation.schema}}].[{{relation.identifier}}] + ({{listColumns}}) SELECT {{listColumns}} FROM [{{tmp_relation.schema}}].[{{tmp_relation.identifier}}] + + {%- else %} + EXEC('CREATE TABLE [{{relation.database}}].[{{relation.schema}}].[{{relation.identifier}}]WITH(DISTRIBUTION = {{dist}},{{index}}) AS (SELECT * FROM [{{tmp_relation.database}}].[{{tmp_relation.schema}}].[{{tmp_relation.identifier}}]);'); + {% endif %} {{ synapse__drop_relation_script(tmp_relation) }} diff --git a/dbt/include/synapse/macros/materializations/models/table/create_table_constraints.sql b/dbt/include/synapse/macros/materializations/models/table/create_table_constraints.sql new file mode 100644 index 00000000..7ae3650e --- /dev/null +++ b/dbt/include/synapse/macros/materializations/models/table/create_table_constraints.sql @@ -0,0 +1,17 @@ +{% macro synapse__build_columns_constraints(relation) %} + {# loop through user_provided_columns to create DDL with data types and constraints #} + {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%} + ( + {% for c in raw_column_constraints -%} + {{ c }}{{ "," if not loop.last }} + {% endfor %} + ) +{% endmacro %} + +{% macro synapse__build_model_constraints(relation) %} + {# loop through user_provided_columns to create DDL with data types and constraints #} + {%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%} + {% for c in raw_model_constraints -%} + alter table {{ relation.include(database=False) }} {{c}}; + {% endfor -%} +{% endmacro %} diff --git a/dbt/include/synapse/macros/materializations/models/view/create_view_as.sql b/dbt/include/synapse/macros/materializations/models/view/create_view_as.sql index b4a93ec5..7deee62e 100644 --- a/dbt/include/synapse/macros/materializations/models/view/create_view_as.sql +++ b/dbt/include/synapse/macros/materializations/models/view/create_view_as.sql @@ -1,4 +1,15 @@ {% macro synapse__create_view_as(relation, sql) -%} - create view {{ relation.include(database=False) }} as - {{ sql }} + + {%- set temp_view_sql = sql.replace("'", "''") -%} + + {% set contract_config = config.get('contract') %} + + {{exceptions.warn("Model contracts cannot be enforced by !")}} + + {% if contract_config.enforced %} + {{ get_assert_columns_equivalent(sql) }} + {%- endif %} + + EXEC('create view {{ relation.include(database=False) }} as {{ temp_view_sql }};'); + {% endmacro %} From d7f11a4a33402d4d4d91d9016fafc4a09513bf7b Mon Sep 17 00:00:00 2001 From: nszoni Date: Fri, 9 Feb 2024 17:39:37 +0100 Subject: [PATCH 17/41] add constraint related functional tests --- tests/functional/adapter/test_constraints.py | 582 +++++++++++++++++++ 1 file changed, 582 insertions(+) create mode 100644 tests/functional/adapter/test_constraints.py diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py new file mode 100644 index 00000000..1e9e4d08 --- /dev/null +++ b/tests/functional/adapter/test_constraints.py @@ -0,0 +1,582 @@ +import pytest +from dbt.tests.adapter.constraints.fixtures import ( + foreign_key_model_sql, + model_data_type_schema_yml, + my_incremental_model_sql, + my_model_data_type_sql, + my_model_incremental_with_nulls_sql, + my_model_incremental_wrong_name_sql, + my_model_incremental_wrong_order_depends_on_fk_sql, + my_model_incremental_wrong_order_sql, + my_model_sql, + my_model_view_wrong_name_sql, + my_model_view_wrong_order_sql, + my_model_with_nulls_sql, + my_model_wrong_name_sql, + my_model_wrong_order_depends_on_fk_sql, + my_model_wrong_order_sql, +) +from dbt.tests.adapter.constraints.test_constraints import ( + BaseConstraintsRuntimeDdlEnforcement, + BaseModelConstraintsRuntimeEnforcement, +) +from dbt.tests.util import ( + get_manifest, + relation_from_name, + run_dbt, + run_dbt_and_capture, + write_file, +) + +model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + constraints: + - type: primary_key + columns: [id] + name: pk_my_model_pk + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + - type: primary_key + - type: unique + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) + - name: my_model_error + config: + contract: + enforced: true + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + - type: primary_key + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) + - name: my_model_wrong_order + config: + contract: + enforced: true + constraints: + - type: unique + columns: [id] + name: uk_my_model_pk + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + - type: primary_key + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) + - name: my_model_wrong_name + config: + contract: + enforced: true + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + - type: primary_key + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) +""" + +model_fk_constraint_schema_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) + - name: my_model_error + config: + contract: + enforced: true + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + - type: primary_key + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) + - name: my_model_wrong_order + config: + contract: + enforced: true + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + - type: primary_key + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) + - name: my_model_wrong_name + config: + contract: + enforced: true + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + - type: primary_key + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) + - name: foreign_key_model + config: + contract: + enforced: true + constraints: + - type: primary_key + columns: [id] + name: pk_my_ref_model_id + - type: unique + name: uk_my_ref_model_id + columns: [id] + columns: + - name: id + data_type: int + constraints: + - type: not_null +""" + +constrained_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + constraints: + - type: primary_key + columns: [ id ] + name: strange_pk_requirement_my_model + - type: unique + columns: [ color, date_day ] + name: strange_uniqueness_requirement_my_model + - type: foreign_key + columns: [ id ] + expression: {schema}.foreign_key_model (id) + name: strange_pk_fk_requirement_my_model + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) + - name: foreign_key_model + config: + contract: + enforced: true + constraints: + - type: primary_key + columns: [ id ] + name: strange_pk_requirement_fk_my_model + - type: unique + columns: [ id ] + name: fk_id_uniqueness_requirement + columns: + - name: id + data_type: int + constraints: + - type: not_null +""" + + +class BaseConstraintsColumnsEqual: + """ + dbt should catch these mismatches during its "preflight" checks. + """ + + @pytest.fixture + def string_type(self): + return "varchar" + + @pytest.fixture + def int_type(self): + return "int" + + @pytest.fixture + def schema_string_type(self, string_type): + return string_type + + @pytest.fixture + def schema_int_type(self, int_type): + return int_type + + @pytest.fixture + def data_types(self, schema_int_type, int_type, string_type): + # sql_column_value, schema_data_type, error_data_type + return [ + ["1", schema_int_type, int_type], + ["'1'", string_type, string_type], + ["cast('2019-01-01' as date)", "date", "date"], + ["cast(1 as bit)", "bit", "bit"], + ["cast('2013-11-03 00:00:00.000000' as datetime2(6))", "datetime2(6)", "datetime2(6)"], + ["cast(1 as decimal(5,2))", "decimal", "decimal"], + ] + + def test__constraints_wrong_column_order(self, project): + # This no longer causes an error, since we enforce yaml column order + run_dbt(["run", "-s", "my_model_wrong_order"], expect_pass=True) + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model_wrong_order" + my_model_config = manifest.nodes[model_id].config + contract_actual_config = my_model_config.contract + + assert contract_actual_config.enforced is True + + def test__constraints_wrong_column_names(self, project, string_type, int_type): + _, log_output = run_dbt_and_capture( + ["run", "-s", "my_model_wrong_name"], expect_pass=False + ) + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model_wrong_name" + my_model_config = manifest.nodes[model_id].config + contract_actual_config = my_model_config.contract + + assert contract_actual_config.enforced is True + + expected = ["id", "error", "missing in definition", "missing in contract"] + assert all([(exp in log_output or exp.upper() in log_output) for exp in expected]) + + def test__constraints_wrong_column_data_types( + self, project, string_type, int_type, schema_string_type, schema_int_type, data_types + ): + for sql_column_value, schema_data_type, error_data_type in data_types: + # Write parametrized data_type to sql file + write_file( + my_model_data_type_sql.format(sql_value=sql_column_value), + "models", + "my_model_data_type.sql", + ) + + # Write wrong data_type to corresponding schema file + # Write integer type for all schema yaml values except when testing integer type itself + wrong_schema_data_type = ( + schema_int_type + if schema_data_type.upper() != schema_int_type.upper() + else schema_string_type + ) + wrong_schema_error_data_type = ( + int_type if schema_data_type.upper() != schema_int_type.upper() else string_type + ) + write_file( + model_data_type_schema_yml.format(data_type=wrong_schema_data_type), + "models", + "constraints_schema.yml", + ) + + results, log_output = run_dbt_and_capture( + ["run", "-s", "my_model_data_type"], expect_pass=False + ) + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model_data_type" + my_model_config = manifest.nodes[model_id].config + contract_actual_config = my_model_config.contract + + assert contract_actual_config.enforced is True + expected = [ + "wrong_data_type_column_name", + error_data_type, + wrong_schema_error_data_type, + "data type mismatch", + ] + assert all([(exp in log_output or exp.upper() in log_output) for exp in expected]) + + def test__constraints_correct_column_data_types(self, project, data_types): + for sql_column_value, schema_data_type, _ in data_types: + # Write parametrized data_type to sql file + write_file( + my_model_data_type_sql.format(sql_value=sql_column_value), + "models", + "my_model_data_type.sql", + ) + # Write correct data_type to corresponding schema file + write_file( + model_data_type_schema_yml.format(data_type=schema_data_type), + "models", + "constraints_schema.yml", + ) + + run_dbt(["run", "-s", "my_model_data_type"]) + + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model_data_type" + my_model_config = manifest.nodes[model_id].config + contract_actual_config = my_model_config.contract + + assert contract_actual_config.enforced is True + + +class BaseConstraintsRuntimeDdlEnforcement(BaseConstraintsRuntimeDdlEnforcement): + """ + These constraints pass muster for dbt's preflight checks. Make sure they're + passed into the DDL statement. If they don't match up with the underlying data, + the data platform should raise an error at runtime. + """ + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_wrong_order_depends_on_fk_sql, + "foreign_key_model.sql": foreign_key_model_sql, + "constraints_schema.yml": model_fk_constraint_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self): + return """ + if object_id is not null begin drop view end + if object_id is not null begin drop table end + exec('create view as -- depends_on: + select ''blue'' as color,1 as id,''2019-01-01'' as date_day;'); + create table (id int not null,color varchar(100),date_day varchar(100)) + with(distribution = round_robin,heap) + insert into ([id],[color],[date_day]) + select [id],[color],[date_day] from + if object_id is not null begin drop view end + """ + + +class BaseIncrementalConstraintsRuntimeDdlEnforcement(BaseConstraintsRuntimeDdlEnforcement): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_incremental_wrong_order_depends_on_fk_sql, + "foreign_key_model.sql": foreign_key_model_sql, + "constraints_schema.yml": model_fk_constraint_schema_yml, + } + + +class BaseModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement): + """ + These model-level constraints pass muster for dbt's preflight checks. Make sure they're + passed into the DDL statement. If they don't match up with the underlying data, + the data platform should raise an error at runtime. + """ + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_wrong_order_depends_on_fk_sql, + "foreign_key_model.sql": foreign_key_model_sql, + "constraints_schema.yml": constrained_model_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self): + return """ + if object_id is not null begin drop view end + if object_id is not null begin drop table end + exec('create view as -- depends_on: + select ''blue'' as color,1 as id,''2019-01-01'' as date_day;'); + create table (id int not null,color varchar(100),date_day varchar(100)) + with(distribution = round_robin,heap) + alter table add constraint + primary key nonclustered(id)not enforced; + alter table add constraint + unique nonclustered(color,date_day)not enforced; + insert into ([id],[color],[date_day]) + select [id],[color],[date_day] from + if object_id is not null begin drop view end + """ + + +class BaseTableConstraintsColumnsEqual(BaseConstraintsColumnsEqual): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_wrong_order_sql, + "my_model_wrong_name.sql": my_model_wrong_name_sql, + "constraints_schema.yml": model_schema_yml, + } + + +class BaseViewConstraintsColumnsEqual(BaseConstraintsColumnsEqual): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_view_wrong_order_sql, + "my_model_wrong_name.sql": my_model_view_wrong_name_sql, + "constraints_schema.yml": model_schema_yml, + } + + +class BaseIncrementalConstraintsColumnsEqual(BaseConstraintsColumnsEqual): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_incremental_wrong_order_sql, + "my_model_wrong_name.sql": my_model_incremental_wrong_name_sql, + "constraints_schema.yml": model_schema_yml, + } + + +class BaseConstraintsRollback: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "constraints_schema.yml": model_schema_yml, + } + + @pytest.fixture(scope="class") + def null_model_sql(self): + return my_model_with_nulls_sql + + @pytest.fixture(scope="class") + def expected_color(self): + return "blue" + + @pytest.fixture(scope="class") + def expected_error_messages(self): + return [ + "Cannot insert the value NULL into column", + "column does not allow nulls", + "There is already an object", + ] + + def assert_expected_error_messages(self, error_message, expected_error_messages): + assert any(msg in error_message for msg in expected_error_messages) + + def test__constraints_enforcement_rollback( + self, project, expected_color, expected_error_messages, null_model_sql + ): + results = run_dbt(["run", "-s", "my_model"]) + assert len(results) == 1 + + # Make a contract-breaking change to the model + write_file(null_model_sql, "models", "my_model.sql") + + failing_results = run_dbt(["run", "-s", "my_model"], expect_pass=False) + assert len(failing_results) == 1 + + # Verify the previous table still exists + relation = relation_from_name(project.adapter, "my_model") + old_model_exists_sql = f"select * from {relation}" + old_model_exists = project.run_sql(old_model_exists_sql, fetch="all") + assert len(old_model_exists) == 1 + assert old_model_exists[0][1] == expected_color + + # Confirm this model was contracted + # TODO: is this step really necessary? + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model" + my_model_config = manifest.nodes[model_id].config + contract_actual_config = my_model_config.contract + assert contract_actual_config.enforced is True + + # Its result includes the expected error messages + self.assert_expected_error_messages(failing_results[0].message, expected_error_messages) + + +class BaseIncrementalConstraintsRollback(BaseConstraintsRollback): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_incremental_model_sql, + "constraints_schema.yml": model_schema_yml, + } + + @pytest.fixture(scope="class") + def null_model_sql(self): + return my_model_incremental_with_nulls_sql + + +class TestTableConstraintsRuntimeDdlEnforcementSynapse(BaseConstraintsRuntimeDdlEnforcement): + pass + + +class TestIncrementalConstraintsRuntimeDdlEnforcementSynapse( + BaseIncrementalConstraintsRuntimeDdlEnforcement +): + pass + + +class TestModelConstraintsRuntimeEnforcementSynapse(BaseModelConstraintsRuntimeEnforcement): + pass + + +class TestTableConstraintsColumnsEqualSynapse(BaseTableConstraintsColumnsEqual): + pass + + +class TestViewConstraintsColumnsEquaSynapse(BaseViewConstraintsColumnsEqual): + pass + + +class TestIncrementalConstraintsColumnsEqualSynapse(BaseIncrementalConstraintsColumnsEqual): + pass + + +class TestTableConstraintsRollbackSynapse(BaseConstraintsRollback): + pass + + +class TestIncrementalConstraintsRollbackSynapse(BaseIncrementalConstraintsRollback): + pass From dc8543483bc37dc11aeadb0b10f880dcea5da6df Mon Sep 17 00:00:00 2001 From: nszoni Date: Fri, 9 Feb 2024 17:41:38 +0100 Subject: [PATCH 18/41] update dependencies --- dev_requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index 802f4848..c91ec5d6 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,13 +1,13 @@ -pytest==7.2.0 +pytest==7.4.0 twine==4.0.2 -wheel==0.40.0 +wheel==0.41.1 pre-commit==2.21.0;python_version<"3.8" -pre-commit==3.3.1;python_version>="3.8" +pre-commit==3.3.3;python_version>="3.8" pytest-dotenv==0.5.2 dbt-tests-adapter~=1.5.9 pyodbc==4.0.39 --no-binary :all: aiohttp==3.8.3 azure-mgmt-synapse==2.0.0 flaky==3.7.0 -pytest-xdist==3.3.0 +pytest-xdist==3.3.1 -e . From 90c6fffe22b37be17f98f3fda528aa499d770392 Mon Sep 17 00:00:00 2001 From: nszoni Date: Fri, 9 Feb 2024 17:43:09 +0100 Subject: [PATCH 19/41] add caching testing suite --- tests/functional/adapter/test_caching.py | 117 +++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 tests/functional/adapter/test_caching.py diff --git a/tests/functional/adapter/test_caching.py b/tests/functional/adapter/test_caching.py new file mode 100644 index 00000000..a517c58f --- /dev/null +++ b/tests/functional/adapter/test_caching.py @@ -0,0 +1,117 @@ +import pytest +from dbt.tests.util import run_dbt + +model_sql = """ +{{ + config( + materialized='table' + ) +}} +select 1 as id +""" + +another_schema_model_sql = """ +{{ + config( + materialized='table', + schema='another_schema' + ) +}} +select 1 as id +""" + + +class BaseCachingTest: + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + "quoting": { + "identifier": False, + "schema": False, + }, + } + + def run_and_inspect_cache(self, project, run_args=None): + run_dbt(run_args) + + # the cache was empty at the start of the run. + # the model materialization returned an unquoted relation and added to the cache. + adapter = project.adapter + assert len(adapter.cache.relations) == 1 + relation = list(adapter.cache.relations).pop() + assert relation.schema == project.test_schema + assert relation.schema == project.test_schema.lower() + + # on the second run, dbt will find a relation in the database during cache population. + # this relation will be quoted, because list_relations_without_caching (by default) uses + # quote_policy = {"database": True, "schema": True, "identifier": True} + # when adding relations to the cache. + run_dbt(run_args) + adapter = project.adapter + assert len(adapter.cache.relations) == 1 + second_relation = list(adapter.cache.relations).pop() + + # perform a case-insensitive + quote-insensitive comparison + for key in ["database", "schema", "identifier"]: + assert getattr(relation, key).lower() == getattr(second_relation, key).lower() + + def test_cache(self, project): + self.run_and_inspect_cache(project, run_args=["run"]) + + +class BaseCachingLowercaseModel(BaseCachingTest): + @pytest.fixture(scope="class") + def models(self): + return { + "model.sql": model_sql, + } + + +class BaseCachingUppercaseModel(BaseCachingTest): + @pytest.fixture(scope="class") + def models(self): + return { + "MODEL.sql": model_sql, + } + + +class BaseCachingSelectedSchemaOnly(BaseCachingTest): + @pytest.fixture(scope="class") + def models(self): + return { + "model.sql": model_sql, + "another_schema_model.sql": another_schema_model_sql, + } + + def test_cache(self, project): + # this should only cache the schema containing the selected model + run_args = ["--cache-selected-only", "run", "--select", "model"] + self.run_and_inspect_cache(project, run_args) + + +class TestNoPopulateCache(BaseCachingTest): + @pytest.fixture(scope="class") + def models(self): + return { + "model.sql": model_sql, + } + + def test_cache(self, project): + # --no-populate-cache still allows the cache to populate all relations + # under a schema, so the behavior here remains the same as other tests + run_args = ["--no-populate-cache", "run"] + self.run_and_inspect_cache(project, run_args) + + +class TestCachingLowerCaseModelSynapse(BaseCachingLowercaseModel): + pass + + +@pytest.mark.skip(reason="Synapse DW does not support Case Insensivity.") +class TestCachingUppercaseModelSynapse(BaseCachingUppercaseModel): + pass + + +class TestCachingSelectedSchemaOnlySynapse(BaseCachingSelectedSchemaOnly): + pass From d259ba517ddfdefc4a6ac6c3cc9754d1be0526be Mon Sep 17 00:00:00 2001 From: nszoni Date: Mon, 12 Feb 2024 09:45:18 +0100 Subject: [PATCH 20/41] add copy uppercase tests --- .../functional/adapter/test_copy_uppercase.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 tests/functional/adapter/test_copy_uppercase.py diff --git a/tests/functional/adapter/test_copy_uppercase.py b/tests/functional/adapter/test_copy_uppercase.py new file mode 100644 index 00000000..8717b752 --- /dev/null +++ b/tests/functional/adapter/test_copy_uppercase.py @@ -0,0 +1,66 @@ +import pytest +from conftest import _profile_ci_azure_auto, _profile_user, _profile_user_azure +from dbt.tests.adapter.simple_copy.fixtures import ( + _MODELS__ADVANCED_INCREMENTAL, + _MODELS__COMPOUND_SORT, + _MODELS__DISABLED, + _MODELS__EMPTY, + _MODELS__INCREMENTAL, + _MODELS__INTERLEAVED_SORT, + _MODELS__MATERIALIZED, + _MODELS__VIEW_MODEL, + _MODELS_GET_AND_REF_UPPERCASE, + _PROPERTIES__SCHEMA_YML, + _SEEDS__SEED_INITIAL, +) +from dbt.tests.util import check_relations_equal, run_dbt + + +def dbt_profile_target(request): + profile = request.config.getoption("--profile") + + if profile == "ci_azure_auto": + return _profile_ci_azure_auto() + if profile == "user": + return _profile_user() + if profile == "user_azure": + return _profile_user_azure() + + +class TestSimpleCopyUppercase: + @pytest.fixture(scope="class") + def models(self): + return { + "ADVANCED_INCREMENTAL.sql": _MODELS__ADVANCED_INCREMENTAL, + "COMPOUND_SORT.sql": _MODELS__COMPOUND_SORT, + "DISABLED.sql": _MODELS__DISABLED, + "EMPTY.sql": _MODELS__EMPTY, + "GET_AND_REF.sql": _MODELS_GET_AND_REF_UPPERCASE, + "INCREMENTAL.sql": _MODELS__INCREMENTAL, + "INTERLEAVED_SORT.sql": _MODELS__INTERLEAVED_SORT, + "MATERIALIZED.sql": _MODELS__MATERIALIZED, + "VIEW_MODEL.sql": _MODELS__VIEW_MODEL, + } + + @pytest.fixture(scope="class") + def properties(self): + return { + "schema.yml": _PROPERTIES__SCHEMA_YML, + } + + @pytest.fixture(scope="class") + def seeds(self): + return {"seed.csv": _SEEDS__SEED_INITIAL} + + def test_simple_copy_uppercase(self, project): + # Load the seed file and check that it worked + results = run_dbt(["seed"]) + assert len(results) == 1 + + # Run the project and ensure that all the models loaded + results = run_dbt() + assert len(results) == 7 + + check_relations_equal( + project.adapter, ["seed", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED", "GET_AND_REF"] + ) From 63c8512eb7a9d833e22d2fbd6350a593aa26a755 Mon Sep 17 00:00:00 2001 From: nszoni Date: Mon, 12 Feb 2024 10:26:14 +0100 Subject: [PATCH 21/41] add simple copy and empty run tests --- tests/functional/adapter/test_simple_copy.py | 108 +++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 tests/functional/adapter/test_simple_copy.py diff --git a/tests/functional/adapter/test_simple_copy.py b/tests/functional/adapter/test_simple_copy.py new file mode 100644 index 00000000..57cae1d7 --- /dev/null +++ b/tests/functional/adapter/test_simple_copy.py @@ -0,0 +1,108 @@ +from pathlib import Path + +import pytest +from dbt.tests.adapter.simple_copy.fixtures import _SEEDS__SEED_UPDATE +from dbt.tests.adapter.simple_copy.test_simple_copy import SimpleCopySetup +from dbt.tests.fixtures.project import TestProjInfo +from dbt.tests.util import check_relations_equal, rm_file, run_dbt, write_file + + +class SynapseTestProjInfo(TestProjInfo): + # This return a dictionary of table names to 'view' or 'table' values. + # Override class because Synapse doesnt have 'ILIKE' + def synapse_get_tables_in_schema(self): + sql = """ + select table_name, + case when table_type = 'BASE TABLE' then 'table' + when table_type = 'VIEW' then 'view' + else table_type + end as materialization + from information_schema.tables + where {} + order by table_name + """ + sql = sql.format("{} like '{}'".format("table_schema", self.test_schema)) + result = self.run_sql(sql, fetch="all") + return {model_name: materialization for (model_name, materialization) in result} + + +# create new project fixture replacing the syntax-incompatible method +@pytest.fixture +def synapse_project(project): + # Replace the original class with the new one + project.__class__ = SynapseTestProjInfo + + return project + + +class SimpleCopyBase(SimpleCopySetup): + def test_simple_copy(self, synapse_project): + # Load the seed file and check that it worked + results = run_dbt(["seed"]) + assert len(results) == 1 + + # Run the synapse_project and ensure that all the models loaded + results = run_dbt() + assert len(results) == 7 + check_relations_equal( + synapse_project.adapter, + ["seed", "view_model", "incremental", "materialized", "get_and_ref"], + ) + + # Change the seed.csv file and see if everything is the same, + # i.e. everything has been updated + main_seed_file = synapse_project.project_root / Path("seeds") / Path("seed.csv") + rm_file(main_seed_file) + write_file(_SEEDS__SEED_UPDATE, main_seed_file) + results = run_dbt(["seed"]) + assert len(results) == 1 + results = run_dbt() + assert len(results) == 7 + check_relations_equal( + synapse_project.adapter, + ["seed", "view_model", "incremental", "materialized", "get_and_ref"], + ) + + @pytest.mark.skip(reason="We are not supporting materialized views yet") + def test_simple_copy_with_materialized_views(self, synapse_project): + synapse_project.run_sql( + f"create table {synapse_project.test_schema}.unrelated_table (id int)" + ) + sql = f""" + create materialized view {synapse_project.test_schema}.unrelated_materialized_view as ( + select * from {synapse_project.test_schema}.unrelated_table + ) + """ + synapse_project.run_sql(sql) + sql = f""" + create view {synapse_project.test_schema}.unrelated_view as ( + select * from {synapse_project.test_schema}.unrelated_materialized_view + ) + """ + synapse_project.run_sql(sql) + results = run_dbt(["seed"]) + assert len(results) == 1 + results = run_dbt() + assert len(results) == 7 + + +class EmptyModelsArentRunBaseSynapse(SimpleCopySetup): + def test_dbt_doesnt_run_empty(self, synapse_project): + results = run_dbt(["seed"]) + assert len(results) == 1 + results = run_dbt() + assert len(results) == 7 + + # Overwriting the original method with the custom implementation + tables = synapse_project.synapse_get_tables_in_schema() + + assert "empty" not in tables.keys() + assert "disabled" not in tables.keys() + + +class TestSimpleCopyBaseSynapse(SimpleCopyBase): + pass + + +class TestEmptyModelsArentRunSynapse(EmptyModelsArentRunBaseSynapse): + pass From a46b5867c04609ecb30f5847b8fd71f49a705ae4 Mon Sep 17 00:00:00 2001 From: artc95 Date: Mon, 12 Feb 2024 14:05:52 +0100 Subject: [PATCH 22/41] fix synapse_adapter.py to import _make_ref_key_msg (instead of ..._dict, not available in dbt-core v1.4.9, see https://github.com/dbt-labs/dbt-core/compare/v1.4.9...v1.7.6) --- dbt/adapters/synapse/synapse_adapter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/synapse/synapse_adapter.py b/dbt/adapters/synapse/synapse_adapter.py index 5c02b579..9e6dfddb 100644 --- a/dbt/adapters/synapse/synapse_adapter.py +++ b/dbt/adapters/synapse/synapse_adapter.py @@ -1,5 +1,5 @@ from dbt.adapters.base.relation import BaseRelation -from dbt.adapters.cache import _make_ref_key_dict +from dbt.adapters.cache import _make_ref_key_msg from dbt.adapters.fabric import FabricAdapter from dbt.adapters.sql.impl import CREATE_SCHEMA_MACRO_NAME from dbt.events.functions import fire_event @@ -13,7 +13,7 @@ class SynapseAdapter(FabricAdapter): def create_schema(self, relation: BaseRelation) -> None: relation = relation.without_identifier() - fire_event(SchemaCreation(relation=_make_ref_key_dict(relation))) + fire_event(SchemaCreation(relation=_make_ref_key_msg(relation))) macro_name = CREATE_SCHEMA_MACRO_NAME kwargs = { "relation": relation, From 7fadfffd313ca4c14a141813d9b1b7924645bd3e Mon Sep 17 00:00:00 2001 From: artc95 Date: Mon, 12 Feb 2024 15:20:47 +0100 Subject: [PATCH 23/41] alert to remove pyodbc from dev_requirements.txt, if using Windows --- CHANGELOG.md | 1 + CONTRIBUTING.md | 2 +- dev_requirements.txt | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e3773db..193369a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ #### Under the hood * Switch dependency from dbt-sqlserver to dbt-fabric (per https://github.com/dbt-msft/dbt-sqlserver/issues/441) + * if Windows, remove `pyodbc==4.0.39 --no-binary :all:` in dev_requirements.txt ## v1.4.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cd5ccd52..fb39952f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,7 +16,7 @@ pyenv activate dbt-synapse Install the development dependencies and pre-commit and get information about possible make commands: ```shell -make dev +make dev # if Windows, remove `pyodbc==4.0.39 --no-binary :all:` in dev_requirements.txt make help ``` diff --git a/dev_requirements.txt b/dev_requirements.txt index 02e93bd0..06071e8d 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -5,7 +5,7 @@ pre-commit==2.21.0;python_version<"3.8" pre-commit==3.3.1;python_version>="3.8" pytest-dotenv==0.5.2 dbt-tests-adapter~=1.4.9 -pyodbc==4.0.39 --no-binary :all: +pyodbc==4.0.39 --no-binary :all: # if Windows, remove aiohttp==3.8.3 azure-mgmt-synapse==2.0.0 flaky==3.7.0 From 18da1725d0b841a3b2611349e477a816327e70ec Mon Sep 17 00:00:00 2001 From: nszoni Date: Mon, 12 Feb 2024 15:26:29 +0100 Subject: [PATCH 24/41] add run hooks tests --- tests/functional/adapter/test_run_hooks.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tests/functional/adapter/test_run_hooks.py diff --git a/tests/functional/adapter/test_run_hooks.py b/tests/functional/adapter/test_run_hooks.py new file mode 100644 index 00000000..0559251f --- /dev/null +++ b/tests/functional/adapter/test_run_hooks.py @@ -0,0 +1,9 @@ +from dbt.tests.adapter.hooks.test_run_hooks import TestAfterRunHooks, TestPrePostRunHooks + + +class TestPrePostRunHooks(TestPrePostRunHooks): + pass + + +class TestAfterRunHooks(TestAfterRunHooks): + pass From 7724115b3107c97a1b137fac0f3c8945554b3c50 Mon Sep 17 00:00:00 2001 From: nszoni Date: Mon, 12 Feb 2024 15:26:59 +0100 Subject: [PATCH 25/41] add run hook seed --- tests/functional/adapter/data/seed_run.sql | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 tests/functional/adapter/data/seed_run.sql diff --git a/tests/functional/adapter/data/seed_run.sql b/tests/functional/adapter/data/seed_run.sql new file mode 100644 index 00000000..3803a9e5 --- /dev/null +++ b/tests/functional/adapter/data/seed_run.sql @@ -0,0 +1,24 @@ +if object_id ('{schema}.on_run_hook','V') is not null +begin +drop table {schema}.on_run_hook +end + +create table {schema}.on_run_hook +( + test_state VARCHAR(100), -- start|end + target_dbname VARCHAR(100), + target_host VARCHAR(100), + target_name VARCHAR(100), + target_schema VARCHAR(100), + target_type VARCHAR(100), + target_user VARCHAR(100), + target_pass VARCHAR(100), + target_threads INTEGER, + run_started_at VARCHAR(100), + invocation_id VARCHAR(100), + thread_id VARCHAR(100) +) +WITH( + DISTRIBUTION = ROUND_ROBIN, + HEAP +) From d88e719735367c2a7cb511796cbf039767a6b871 Mon Sep 17 00:00:00 2001 From: nszoni Date: Mon, 12 Feb 2024 16:44:40 +0100 Subject: [PATCH 26/41] add model hooks tests --- tests/functional/adapter/data/seed_model.sql | 18 + tests/functional/adapter/data/seed_run.sql | 3 +- tests/functional/adapter/test_model_hooks.py | 347 +++++++++++++++++++ 3 files changed, 366 insertions(+), 2 deletions(-) create mode 100644 tests/functional/adapter/data/seed_model.sql create mode 100644 tests/functional/adapter/test_model_hooks.py diff --git a/tests/functional/adapter/data/seed_model.sql b/tests/functional/adapter/data/seed_model.sql new file mode 100644 index 00000000..acf53349 --- /dev/null +++ b/tests/functional/adapter/data/seed_model.sql @@ -0,0 +1,18 @@ +create table {schema}.on_model_hook +( + test_state VARCHAR(100), -- start|end + target_dbname VARCHAR(100), + target_host VARCHAR(100), + target_name VARCHAR(100), + target_schema VARCHAR(100), + target_type VARCHAR(100), + target_user VARCHAR(100), + target_pass VARCHAR(100), + target_threads INTEGER, + run_started_at VARCHAR(100), + invocation_id VARCHAR(100) +) +WITH( + DISTRIBUTION = ROUND_ROBIN, + HEAP +) diff --git a/tests/functional/adapter/data/seed_run.sql b/tests/functional/adapter/data/seed_run.sql index 3803a9e5..e609c188 100644 --- a/tests/functional/adapter/data/seed_run.sql +++ b/tests/functional/adapter/data/seed_run.sql @@ -15,8 +15,7 @@ create table {schema}.on_run_hook target_pass VARCHAR(100), target_threads INTEGER, run_started_at VARCHAR(100), - invocation_id VARCHAR(100), - thread_id VARCHAR(100) + invocation_id VARCHAR(100) ) WITH( DISTRIBUTION = ROUND_ROBIN, diff --git a/tests/functional/adapter/test_model_hooks.py b/tests/functional/adapter/test_model_hooks.py new file mode 100644 index 00000000..05a757d3 --- /dev/null +++ b/tests/functional/adapter/test_model_hooks.py @@ -0,0 +1,347 @@ +from pathlib import Path + +import pytest +from dbt.tests.adapter.hooks.fixtures import ( + models__hooked, + models__hooks, + models__hooks_configured, + models__hooks_kwargs, + models__post, + models__pre, +) +from dbt.tests.adapter.hooks.test_model_hooks import ( + TestDuplicateHooksInConfigs, + TestHooksRefsOnSeeds, + TestPrePostModelHooksOnSeeds, + TestPrePostModelHooksOnSeedsPlusPrefixed, + TestPrePostModelHooksOnSeedsPlusPrefixedWhitespace, + TestPrePostModelHooksOnSnapshots, + TestPrePostSnapshotHooksInConfigKwargs, +) +from dbt.tests.util import run_dbt + +MODEL_PRE_HOOK = """ + insert into {{this.schema}}.on_model_hook ( + test_state, + target_dbname, + target_host, + target_name, + target_schema, + target_type, + target_user, + target_pass, + target_threads, + run_started_at, + invocation_id + ) VALUES ( + 'start', + '{{ target.dbname }}', + '{{ target.host }}', + '{{ target.name }}', + '{{ target.schema }}', + '{{ target.type }}', + '{{ target.user }}', + '{{ target.get("pass", "") }}', + {{ target.threads }}, + '{{ run_started_at }}', + '{{ invocation_id }}' + ) +""" + +MODEL_POST_HOOK = """ + insert into {{this.schema}}.on_model_hook ( + test_state, + target_dbname, + target_host, + target_name, + target_schema, + target_type, + target_user, + target_pass, + target_threads, + run_started_at, + invocation_id + ) VALUES ( + 'end', + '{{ target.dbname }}', + '{{ target.host }}', + '{{ target.name }}', + '{{ target.schema }}', + '{{ target.type }}', + '{{ target.user }}', + '{{ target.get("pass", "") }}', + {{ target.threads }}, + '{{ run_started_at }}', + '{{ invocation_id }}' + ) +""" + + +class BaseTestPrePost(object): + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + project.run_sql_file(project.test_data_dir / Path("seed_model.sql")) + + def get_ctx_vars(self, state, count, project): + fields = [ + "test_state", + "target_dbname", + "target_host", + "target_name", + "target_schema", + "target_threads", + "target_type", + "target_user", + "target_pass", + "run_started_at", + "invocation_id", + ] + field_list = ", ".join(['"{}"'.format(f) for f in fields]) + query = f""" + select {field_list} from {project.test_schema}.on_model_hook where test_state = '{state}' + """ + + vals = project.run_sql(query, fetch="all") + assert len(vals) != 0, "nothing inserted into hooks table" + assert len(vals) >= count, "too few rows in hooks table" + assert len(vals) <= count, "too many rows in hooks table" + return [{k: v for k, v in zip(fields, val)} for val in vals] + + def check_hooks(self, state, project, host, count=1): + ctxs = self.get_ctx_vars(state, count=count, project=project) + for ctx in ctxs: + assert ctx["test_state"] == state + assert ctx["target_dbname"] == "" + assert ctx["target_host"] == "" + assert ctx["target_name"] == "default" + assert ctx["target_schema"] == project.test_schema + assert ctx["target_threads"] == 1 + assert ctx["target_type"] == "synapse" + assert ctx["target_user"] == "None" + assert ctx["target_pass"] == "" + + assert ( + ctx["run_started_at"] is not None and len(ctx["run_started_at"]) > 0 + ), "run_started_at was not set" + assert ( + ctx["invocation_id"] is not None and len(ctx["invocation_id"]) > 0 + ), "invocation_id was not set" + + +class PrePostModelHooksInConfigSetup(BaseTestPrePost): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "macro-paths": ["macros"], + } + + @pytest.fixture(scope="class") + def models(self): + return {"hooks.sql": models__hooks_configured} + + +class TestHookRefs(BaseTestPrePost): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "hooked": { + "post-hook": [ + """ + insert into {{this.schema}}.on_model_hook select + test_state, + '{{ target.dbname }}' as target_dbname, + '{{ target.host }}' as target_host, + '{{ target.name }}' as target_name, + '{{ target.schema }}' as target_schema, + '{{ target.type }}' as target_type, + '{{ target.user }}' as target_user, + '{{ target.get(pass, "") }}' as target_pass, + {{ target.threads }} as target_threads, + '{{ run_started_at }}' as run_started_at, + '{{ invocation_id }}' as invocation_id + from {{ ref('post') }}""".strip() + ], + } + }, + } + } + + @pytest.fixture(scope="class") + def models(self): + return {"hooked.sql": models__hooked, "post.sql": models__post, "pre.sql": models__pre} + + def test_pre_post_model_hooks_refed(self, project, dbt_profile_target): + run_dbt() + self.check_hooks("start", project, dbt_profile_target.get("host", None)) + self.check_hooks("end", project, dbt_profile_target.get("host", None)) + + +class TestPrePostModelHooksOnSeeds(TestPrePostModelHooksOnSeeds): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seed-paths": ["seeds"], + "models": {}, + "seeds": { + "post-hook": [ + "alter table {{ this }} add new_col int", + "update {{ this }} set new_col = 1", + # call any macro to track dependency: + # https://github.com/dbt-labs/dbt-core/issues/6806 + "select cast(null as {{ dbt.type_int() }}) as id", + ], + "quote_columns": False, + }, + } + + +class TestHooksRefsOnSeeds(TestHooksRefsOnSeeds): + pass + + +# +class TestPrePostModelHooksOnSeedsPlusPrefixed(TestPrePostModelHooksOnSeedsPlusPrefixed): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seed-paths": ["seeds"], + "models": {}, + "seeds": { + "+post-hook": [ + "alter table {{ this }} add new_col int", + "update {{ this }} set new_col = 1", + ], + "quote_columns": False, + }, + } + + +class TestPrePostModelHooksOnSeedsPlusPrefixedWhitespace( + TestPrePostModelHooksOnSeedsPlusPrefixedWhitespace +): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seed-paths": ["seeds"], + "models": {}, + "seeds": { + "+post-hook": [ + "alter table {{ this }} add new_col int", + "update {{ this }} set new_col = 1", + ], + "quote_columns": False, + }, + } + + +class TestPrePostModelHooksOnSnapshots(TestPrePostModelHooksOnSnapshots): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seed-paths": ["seeds"], + "snapshot-paths": ["test-snapshots"], + "models": {}, + "snapshots": { + "post-hook": [ + "alter table {{ this }} add new_col int", + "update {{ this }} set new_col = 1", + ] + }, + "seeds": { + "quote_columns": False, + }, + } + + +class TestPrePostModelHooksInConfig(PrePostModelHooksInConfigSetup): + def test_pre_and_post_model_hooks_model(self, project, dbt_profile_target): + run_dbt() + + self.check_hooks("start", project, dbt_profile_target.get("host", None)) + self.check_hooks("end", project, dbt_profile_target.get("host", None)) + + +class TestPrePostModelHooksInConfigKwargs(TestPrePostModelHooksInConfig): + @pytest.fixture(scope="class") + def models(self): + return {"hooks.sql": models__hooks_kwargs} + + +class TestPrePostSnapshotHooksInConfigKwargs(TestPrePostSnapshotHooksInConfigKwargs): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seed-paths": ["seeds"], + "snapshot-paths": ["test-kwargs-snapshots"], + "models": {}, + "snapshots": { + "post-hook": [ + "alter table {{ this }} add new_col int", + "update {{ this }} set new_col = 1", + ] + }, + "seeds": { + "quote_columns": False, + }, + } + + +class TestDuplicateHooksInConfigs(TestDuplicateHooksInConfigs): + pass + + +# vacuum command is removed because not supported in synapse +class TestPrePostModelHooks(BaseTestPrePost): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "pre-hook": [MODEL_PRE_HOOK], + "post-hook": [MODEL_POST_HOOK], + } + } + } + + @pytest.fixture(scope="class") + def models(self): + return {"hooks.sql": models__hooks} + + def test_pre_and_post_run_hooks(self, project, dbt_profile_target): + run_dbt() + self.check_hooks("start", project, dbt_profile_target.get("host", None)) + self.check_hooks("end", project, dbt_profile_target.get("host", None)) + + +class TestPrePostModelHooksInConfigWithCount(PrePostModelHooksInConfigSetup): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "pre-hook": [MODEL_PRE_HOOK], + "post-hook": [MODEL_POST_HOOK], + } + } + } + + def test_pre_and_post_model_hooks_model_and_project(self, project, dbt_profile_target): + run_dbt() + + self.check_hooks("start", project, dbt_profile_target.get("host", None), count=2) + self.check_hooks("end", project, dbt_profile_target.get("host", None), count=2) + + +@pytest.mark.skip(reason="Not supporting underscores config") +class TestPrePostModelHooksUnderscores(TestPrePostModelHooks): + def project_config_update(self): + return { + "models": { + "test": { + "pre_hook": [MODEL_PRE_HOOK], + "post_hook": [MODEL_POST_HOOK], + } + } + } From 0480fe9c6a22037f7432454a5a8468791e8ce570 Mon Sep 17 00:00:00 2001 From: nszoni Date: Tue, 13 Feb 2024 10:41:22 +0100 Subject: [PATCH 27/41] add persist docs note --- .../synapse/macros/adapters/persist_docs.sql | 4 ++++ tests/functional/adapter/test_persist_docs.py | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 dbt/include/synapse/macros/adapters/persist_docs.sql create mode 100644 tests/functional/adapter/test_persist_docs.py diff --git a/dbt/include/synapse/macros/adapters/persist_docs.sql b/dbt/include/synapse/macros/adapters/persist_docs.sql new file mode 100644 index 00000000..091f954d --- /dev/null +++ b/dbt/include/synapse/macros/adapters/persist_docs.sql @@ -0,0 +1,4 @@ +{# Unfortunately adding docs via extended properties is not supported in Synapse only in SQLServer + https://github.com/dbt-msft/dbt-sqlserver/issues/134 + https://learn.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-addextendedproperty-transact-sql?view=sql-server-ver16 + #} diff --git a/tests/functional/adapter/test_persist_docs.py b/tests/functional/adapter/test_persist_docs.py new file mode 100644 index 00000000..e45f90cb --- /dev/null +++ b/tests/functional/adapter/test_persist_docs.py @@ -0,0 +1,21 @@ +import pytest +from dbt.tests.adapter.persist_docs.test_persist_docs import ( + BasePersistDocs, + BasePersistDocsColumnMissing, + BasePersistDocsCommentOnQuotedColumn, +) + + +@pytest.mark.skip(reason="Synapse does not support adding/updating extended properties") +class TestPersistDocsSynapse(BasePersistDocs): + pass + + +@pytest.mark.skip(reason="Synapse does not support adding/updating extended properties") +class TestPersistDocsColumnMissingSynapse(BasePersistDocsColumnMissing): + pass + + +@pytest.mark.skip(reason="Synapse does not support adding/updating extended properties") +class TestPersistDocsCommentOnQuotedColumnSynapse(BasePersistDocsCommentOnQuotedColumn): + pass From d78f17a9a0772525726c772e5f24a1d174e07b45 Mon Sep 17 00:00:00 2001 From: nszoni Date: Tue, 13 Feb 2024 12:30:05 +0100 Subject: [PATCH 28/41] fix run hooks --- tests/functional/adapter/data/seed_run.sql | 8 +- tests/functional/adapter/test_run_hooks.py | 177 ++++++++++++++++++++- 2 files changed, 175 insertions(+), 10 deletions(-) diff --git a/tests/functional/adapter/data/seed_run.sql b/tests/functional/adapter/data/seed_run.sql index e609c188..85a02e26 100644 --- a/tests/functional/adapter/data/seed_run.sql +++ b/tests/functional/adapter/data/seed_run.sql @@ -1,7 +1,5 @@ -if object_id ('{schema}.on_run_hook','V') is not null -begin -drop table {schema}.on_run_hook -end +if object_id ('{schema}.on_run_hook') is not null + drop table {schema}.on_run_hook; create table {schema}.on_run_hook ( @@ -20,4 +18,4 @@ create table {schema}.on_run_hook WITH( DISTRIBUTION = ROUND_ROBIN, HEAP -) +); diff --git a/tests/functional/adapter/test_run_hooks.py b/tests/functional/adapter/test_run_hooks.py index 0559251f..e8baad8d 100644 --- a/tests/functional/adapter/test_run_hooks.py +++ b/tests/functional/adapter/test_run_hooks.py @@ -1,9 +1,176 @@ -from dbt.tests.adapter.hooks.test_run_hooks import TestAfterRunHooks, TestPrePostRunHooks +import os +from pathlib import Path +import pytest +from dbt.tests.adapter.hooks.fixtures import ( + macros__before_and_after, + macros__hook, + macros_missing_column, + models__hooks, + models__missing_column, + seeds__example_seed_csv, +) +from dbt.tests.util import check_table_does_not_exist, run_dbt -class TestPrePostRunHooks(TestPrePostRunHooks): - pass +class TestPrePostRunHooks(object): + @pytest.fixture(scope="function") + def setUp(self, project): + project.run_sql_file(project.test_data_dir / Path("seed_run.sql")) + project.run_sql( + f""" + if object_id ('{ project.test_schema }.schemas') is not null + drop table { project.test_schema }.schemas + """ + ) + project.run_sql( + f""" + if object_id ('{ project.test_schema }.db_schemas') is not null + drop table { project.test_schema }.db_schemas + """ + ) + os.environ["TERM_TEST"] = "TESTING" -class TestAfterRunHooks(TestAfterRunHooks): - pass + @pytest.fixture(scope="class") + def macros(self): + return {"hook.sql": macros__hook, "before-and-after.sql": macros__before_and_after} + + @pytest.fixture(scope="class") + def models(self): + return {"hooks.sql": models__hooks} + + @pytest.fixture(scope="class") + def seeds(self): + return {"example_seed.csv": seeds__example_seed_csv} + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + # The create and drop table statements here validate that these hooks run + # in the same order that they are defined. Drop before create is an error. + # Also check that the table does not exist below. + "on-run-start": [ + "{{ custom_run_hook('start', target, run_started_at, invocation_id) }}", + "create table {{ target.schema }}.start_hook_order_test ( id int )", + "drop table {{ target.schema }}.start_hook_order_test", + "{{ log(env_var('TERM_TEST'), info=True) }}", + ], + "on-run-end": [ + "{{ custom_run_hook('end', target, run_started_at, invocation_id) }}", + "create table {{ target.schema }}.end_hook_order_test ( id int )", + "drop table {{ target.schema }}.end_hook_order_test", + "create table {{ target.schema }}.schemas ( sch varchar(100) )", + """insert into {{ target.schema }}.schemas (sch) values + {% for schema in schemas %}( '{{ schema }}' ) + {% if not loop.last %},{% endif %}{% endfor %}""", + """create table {{ target.schema }}.db_schemas + ( db varchar(100), sch varchar(100) )""", + """insert into {{ target.schema }}.db_schemas (db, sch) values + {% for db, schema in database_schemas %}('{{ db }}', '{{ schema }}' ) + {% if not loop.last %},{% endif %}{% endfor %}""", + ], + "seeds": { + "quote_columns": False, + }, + } + + def get_ctx_vars(self, state, project): + fields = [ + "test_state", + "target_dbname", + "target_host", + "target_name", + "target_schema", + "target_threads", + "target_type", + "target_user", + "target_pass", + "run_started_at", + "invocation_id", + ] + field_list = ", ".join(['"{}"'.format(f) for f in fields]) + query = f""" + select {field_list} from {project.test_schema}.on_run_hook + where test_state = '{state}' + """ + + vals = project.run_sql(query, fetch="all") + assert len(vals) != 0, "nothing inserted into on_run_hook table" + assert len(vals) == 1, "too many rows in hooks table" + ctx = dict([(k, v) for (k, v) in zip(fields, vals[0])]) + + return ctx + + def assert_used_schemas(self, project): + schemas_query = "select * from {}.schemas".format(project.test_schema) + results = project.run_sql(schemas_query, fetch="all") + assert len(results) == 1 + assert results[0][0] == project.test_schema + + db_schemas_query = "select * from {}.db_schemas".format(project.test_schema) + results = project.run_sql(db_schemas_query, fetch="all") + assert len(results) == 1 + assert results[0][0] == project.database + assert results[0][1] == project.test_schema + + def check_hooks(self, state, project, host): + ctx = self.get_ctx_vars(state, project) + + assert ctx["test_state"] == state + assert ctx["target_dbname"] == "" + assert ctx["target_host"] == "" + assert ctx["target_name"] == "default" + assert ctx["target_schema"] == project.test_schema + assert ctx["target_threads"] == 1 + assert ctx["target_type"] == "synapse" + assert ctx["target_user"] == "None" + assert ctx["target_pass"] == "" + + assert ( + ctx["run_started_at"] is not None and len(ctx["run_started_at"]) > 0 + ), "run_started_at was not set" + assert ( + ctx["invocation_id"] is not None and len(ctx["invocation_id"]) > 0 + ), "invocation_id was not set" + + def test_pre_and_post_run_hooks(self, setUp, project, dbt_profile_target): + run_dbt(["run"]) + + self.check_hooks("start", project, dbt_profile_target.get("host", None)) + self.check_hooks("end", project, dbt_profile_target.get("host", None)) + + check_table_does_not_exist(project.adapter, "start_hook_order_test") + check_table_does_not_exist(project.adapter, "end_hook_order_test") + self.assert_used_schemas(project) + + def test_pre_and_post_seed_hooks(self, setUp, project, dbt_profile_target): + run_dbt(["seed"]) + + self.check_hooks("start", project, dbt_profile_target.get("host", None)) + self.check_hooks("end", project, dbt_profile_target.get("host", None)) + + check_table_does_not_exist(project.adapter, "start_hook_order_test") + check_table_does_not_exist(project.adapter, "end_hook_order_test") + self.assert_used_schemas(project) + + +class TestAfterRunHooks(object): + @pytest.fixture(scope="class") + def macros(self): + return {"temp_macro.sql": macros_missing_column} + + @pytest.fixture(scope="class") + def models(self): + return {"test_column.sql": models__missing_column} + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + # The create and drop table statements here validate that these hooks run + # in the same order that they are defined. Drop before create is an error. + # Also check that the table does not exist below. + "on-run-start": "- {{ export_table_check() }}" + } + + def test_missing_column_pre_hook(self, project): + run_dbt(["run"], expect_pass=False) From 7e9e59103cfd9e6ca67e5d5fc8350ed35362d8bb Mon Sep 17 00:00:00 2001 From: nszoni Date: Tue, 13 Feb 2024 13:03:52 +0100 Subject: [PATCH 29/41] add column type adapter tests --- dbt/adapters/synapse/__init__.py | 9 +- dbt/adapters/synapse/synapse_adapter.py | 2 + dbt/adapters/synapse/synapse_column.py | 5 + tests/functional/adapter/test_column_types.py | 126 ++++++++++++++++++ 4 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 dbt/adapters/synapse/synapse_column.py create mode 100644 tests/functional/adapter/test_column_types.py diff --git a/dbt/adapters/synapse/__init__.py b/dbt/adapters/synapse/__init__.py index 6a0d96fd..61f6d4bb 100644 --- a/dbt/adapters/synapse/__init__.py +++ b/dbt/adapters/synapse/__init__.py @@ -1,6 +1,7 @@ from dbt.adapters.base import AdapterPlugin from dbt.adapters.synapse.synapse_adapter import SynapseAdapter +from dbt.adapters.synapse.synapse_column import SynapseColumn from dbt.adapters.synapse.synapse_connection_manager import SynapseConnectionManager from dbt.adapters.synapse.synapse_credentials import SynapseCredentials from dbt.include import synapse @@ -12,4 +13,10 @@ dependencies=["fabric"], ) -__all__ = ["Plugin", "SynapseConnectionManager", "SynapseAdapter", "SynapseCredentials"] +__all__ = [ + "Plugin", + "SynapseConnectionManager", + "SynapseColumn", + "SynapseAdapter", + "SynapseCredentials", +] diff --git a/dbt/adapters/synapse/synapse_adapter.py b/dbt/adapters/synapse/synapse_adapter.py index 4b715bcd..c628d02a 100644 --- a/dbt/adapters/synapse/synapse_adapter.py +++ b/dbt/adapters/synapse/synapse_adapter.py @@ -9,11 +9,13 @@ from dbt.events.functions import fire_event from dbt.events.types import SchemaCreation +from dbt.adapters.synapse.synapse_column import SynapseColumn from dbt.adapters.synapse.synapse_connection_manager import SynapseConnectionManager class SynapseAdapter(FabricAdapter): ConnectionManager = SynapseConnectionManager + Column = SynapseColumn def create_schema(self, relation: BaseRelation) -> None: relation = relation.without_identifier() diff --git a/dbt/adapters/synapse/synapse_column.py b/dbt/adapters/synapse/synapse_column.py new file mode 100644 index 00000000..fed4d6ba --- /dev/null +++ b/dbt/adapters/synapse/synapse_column.py @@ -0,0 +1,5 @@ +from dbt.adapters.fabric import FabricColumn + + +class SynapseColumn(FabricColumn): + ... diff --git a/tests/functional/adapter/test_column_types.py b/tests/functional/adapter/test_column_types.py new file mode 100644 index 00000000..455ec252 --- /dev/null +++ b/tests/functional/adapter/test_column_types.py @@ -0,0 +1,126 @@ +import pytest +from dbt.tests.util import run_dbt + +model_sql = """ +select + cast(1 as smallint) as smallint_col, + cast(2 as integer) as int_col, + cast(3 as bigint) as bigint_col, + cast(4.0 as real) as real_col, + cast(5.0 as double precision) as double_col, + cast(6.0 as numeric) as numeric_col, + cast('7' as varchar(20)) as varchar_col +""" + +schema_yml = """ +version: 2 +models: + - name: model + tests: + - is_type: + column_map: + smallint_col: ['integer', 'number'] + int_col: ['integer', 'number'] + bigint_col: ['integer', 'number'] + real_col: ['float', 'number'] + double_col: ['float', 'number'] + numeric_col: ['numeric', 'number'] + varchar_col: ['string', 'not number'] +""" + +macro_test_is_type_sql = """ +{% macro simple_type_check_column(column, check) %} + {% if check == 'string' %} + {{ return(column.is_string()) }} + {% elif check == 'float' %} + {{ return(column.is_float()) }} + {% elif check == 'number' %} + {{ return(column.is_number()) }} + {% elif check == 'numeric' %} + {{ return(column.is_numeric()) }} + {% elif check == 'integer' %} + {{ return(column.is_integer()) }} + {% else %} + {% do exceptions.raise_compiler_error('invalid type check value: ' ~ check) %} + {% endif %} +{% endmacro %} + +{% macro type_check_column(column, type_checks) %} + {% set failures = [] %} + {% for type_check in type_checks %} + {% if type_check.startswith('not ') %} + {% if simple_type_check_column(column, type_check[4:]) %} + {% do log('simple_type_check_column got ', True) %} + {% do failures.append(type_check) %} + {% endif %} + {% else %} + {% if not simple_type_check_column(column, type_check) %} + {% do failures.append(type_check) %} + {% endif %} + {% endif %} + {% endfor %} + {% if (failures | length) > 0 %} + {% do log('column ' ~ column.name ~ ' had failures: ' ~ failures, info=True) %} + {% endif %} + {% do return((failures | length) == 0) %} +{% endmacro %} + +{% test is_type(model, column_map) %} + {% if not execute %} + {{ return(None) }} + {% endif %} + {% if not column_map %} + {% do exceptions.raise_compiler_error('test_is_type must have a column name') %} + {% endif %} + {% set columns = adapter.get_columns_in_relation(model) %} + {% if (column_map | length) != (columns | length) %} + {% set column_map_keys = (column_map | list | string) %} + {% set column_names = (columns | map(attribute='name') | list | string) %} + {% do exceptions.raise_compiler_error('did not get all the columns/all columns not + specified:\n' ~ column_map_keys ~ '\nvs\n' ~ column_names) %} + {% endif %} + {% set bad_columns = [] %} + {% for column in columns %} + {% set column_key = (column.name | lower) %} + {% if column_key in column_map %} + {% set type_checks = column_map[column_key] %} + {% if not type_checks %} + {% do exceptions.raise_compiler_error('no type checks?') %} + {% endif %} + {% if not type_check_column(column, type_checks) %} + {% do bad_columns.append(column.name) %} + {% endif %} + {% else %} + {% do exceptions.raise_compiler_error('column key ' ~ column_key ~ ' + not found in ' ~ (column_map | list | string)) %} + {% endif %} + {% endfor %} + {% do log('bad columns: ' ~ bad_columns, info=True) %} + {% for bad_column in bad_columns %} + select '{{ bad_column }}' as bad_column + {{ 'union all' if not loop.last }} + {% endfor %} + select top 0 1 as nothing +{% endtest %} +""" + + +class BaseColumnTypes: + @pytest.fixture(scope="class") + def macros(self): + return {"test_is_type.sql": macro_test_is_type_sql} + + def run_and_test(self): + results = run_dbt(["run"]) + assert len(results) == 1 + results = run_dbt(["test"]) + assert len(results) == 1 + + +class TestBaseColumnTypesSynapse(BaseColumnTypes): + @pytest.fixture(scope="class") + def models(self): + return {"model.sql": model_sql, "schema.yml": schema_yml} + + def test_run_and_test(self, project): + self.run_and_test() From e1816e0b0221af37b96818eb20a777ded0b21b17 Mon Sep 17 00:00:00 2001 From: nszoni Date: Tue, 13 Feb 2024 13:04:31 +0100 Subject: [PATCH 30/41] add changelog --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f47ea109..c5aa4628 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## v.1.5.0rc1 * Support for [dbt-core 1.5](https://github.com/dbt-labs/dbt-core/releases/tag/v1.5.0) + * Add support for model contracts by adapting `create_table_as` and `create_view_as` macros + * Define supported constraints in `CONSTRAINT_SUPPORT` Adapter class. + * Persist docs via [extended properties]() is [not supported]((https://learn.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-addextendedproperty-transact-sql?view=sql-server-ver16)) in Synapse + * Add adapter tests zones + - caching + - column_types + - constraints + - hooks + - simple_copy ## v1.4.1rc1 From 878fa8da5232c7c9e58ceeb6e8d3aadb931cf79a Mon Sep 17 00:00:00 2001 From: nszoni Date: Tue, 13 Feb 2024 13:06:38 +0100 Subject: [PATCH 31/41] fix changelog links --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5aa4628..faf3c4bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ * Support for [dbt-core 1.5](https://github.com/dbt-labs/dbt-core/releases/tag/v1.5.0) * Add support for model contracts by adapting `create_table_as` and `create_view_as` macros * Define supported constraints in `CONSTRAINT_SUPPORT` Adapter class. - * Persist docs via [extended properties]() is [not supported]((https://learn.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-addextendedproperty-transact-sql?view=sql-server-ver16)) in Synapse + * Persist docs via [extended properties](https://github.com/dbt-msft/dbt-sqlserver/issues/134) is [not supported](https://learn.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-addextendedproperty-transact-sql?view=sql-server-ver16) in Synapse * Add adapter tests zones - caching - column_types From 20e58aac9f325aac22be5334051502095e6b9da2 Mon Sep 17 00:00:00 2001 From: nszoni Date: Tue, 13 Feb 2024 13:39:01 +0100 Subject: [PATCH 32/41] change ref key --- dbt/adapters/synapse/synapse_adapter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/synapse/synapse_adapter.py b/dbt/adapters/synapse/synapse_adapter.py index 389bafea..c628d02a 100644 --- a/dbt/adapters/synapse/synapse_adapter.py +++ b/dbt/adapters/synapse/synapse_adapter.py @@ -2,7 +2,7 @@ from typing import Any, Dict, List, Optional from dbt.adapters.base.relation import BaseRelation -from dbt.adapters.cache import _make_ref_key_msg +from dbt.adapters.cache import _make_ref_key_dict from dbt.adapters.fabric import FabricAdapter from dbt.adapters.sql.impl import CREATE_SCHEMA_MACRO_NAME from dbt.contracts.graph.nodes import ColumnLevelConstraint, ConstraintType @@ -19,7 +19,7 @@ class SynapseAdapter(FabricAdapter): def create_schema(self, relation: BaseRelation) -> None: relation = relation.without_identifier() - fire_event(SchemaCreation(relation=_make_ref_key_msg(relation))) + fire_event(SchemaCreation(relation=_make_ref_key_dict(relation))) macro_name = CREATE_SCHEMA_MACRO_NAME kwargs = { "relation": relation, From 06c8cb07fa2fba8a5b924dc7ea17c46639e94a41 Mon Sep 17 00:00:00 2001 From: nszoni Date: Tue, 13 Feb 2024 17:05:06 +0100 Subject: [PATCH 33/41] clean sqlserver refs --- tests/functional/adapter/test_grants.py | 2 +- tests/functional/adapter/test_new_project.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/adapter/test_grants.py b/tests/functional/adapter/test_grants.py index 026512d0..060087d0 100644 --- a/tests/functional/adapter/test_grants.py +++ b/tests/functional/adapter/test_grants.py @@ -13,7 +13,7 @@ class TestIncrementalGrantsSynapse(BaseIncrementalGrants): pass -class TestInvalidGrantsSQLServer(BaseInvalidGrants): +class TestInvalidGrantsSynapse(BaseInvalidGrants): def grantee_does_not_exist_error(self): return "Cannot find the user" diff --git a/tests/functional/adapter/test_new_project.py b/tests/functional/adapter/test_new_project.py index b5eef440..7a9fadf1 100644 --- a/tests/functional/adapter/test_new_project.py +++ b/tests/functional/adapter/test_new_project.py @@ -62,7 +62,7 @@ """ -class TestNewProjectSQLServer: +class TestNewProjectSynapse: @pytest.fixture(scope="class") def project_config_update(self): return {"name": "my_new_project"} From 194c7d4cd51c4afa034b12d41409115552d8b634 Mon Sep 17 00:00:00 2001 From: nszoni Date: Tue, 13 Feb 2024 17:05:37 +0100 Subject: [PATCH 34/41] overwrite is_integer list --- dbt/adapters/synapse/synapse_column.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/synapse/synapse_column.py b/dbt/adapters/synapse/synapse_column.py index fed4d6ba..37e336cf 100644 --- a/dbt/adapters/synapse/synapse_column.py +++ b/dbt/adapters/synapse/synapse_column.py @@ -2,4 +2,15 @@ class SynapseColumn(FabricColumn): - ... + # extending list of integer types for synapse + def is_integer(self) -> bool: + return self.dtype.lower() in [ + # real types + "smallint", + "bigint", + "tinyint", + "serial", + "bigserial", + "int", + "bit", + ] From cc1aa5bb81855549edbe34a59627ecef951deb7c Mon Sep 17 00:00:00 2001 From: nszoni Date: Tue, 13 Feb 2024 17:06:01 +0100 Subject: [PATCH 35/41] clean dev requirements --- dev_requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index 92d0aeca..7887a355 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -5,7 +5,6 @@ pre-commit==2.21.0;python_version<"3.8" pre-commit==3.3.3;python_version>="3.8" pytest-dotenv==0.5.2 dbt-tests-adapter~=1.5.9 -pyodbc==4.0.39 --no-binary :all: # if Windows, remove aiohttp==3.8.3 azure-mgmt-synapse==2.0.0 flaky==3.7.0 From 5fd1851c59f824f00a61f958d303bdd7c0e9ed3e Mon Sep 17 00:00:00 2001 From: artc95 Date: Wed, 14 Feb 2024 09:52:15 +0100 Subject: [PATCH 36/41] remove pyodbc from dev_requirements.txt, alert about pyodbc for Mac users --- CHANGELOG.md | 3 ++- CONTRIBUTING.md | 2 +- dev_requirements.txt | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 193369a6..6e1b2e67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,8 @@ #### Under the hood * Switch dependency from dbt-sqlserver to dbt-fabric (per https://github.com/dbt-msft/dbt-sqlserver/issues/441) - * if Windows, remove `pyodbc==4.0.39 --no-binary :all:` in dev_requirements.txt + * for Mac users, before running `make dev`, add `pyodbc==4.0.39 --no-binary :all:` in dev_requirements.txt + * about pyodbc "Symbol not found: _SQLAllocHandle" error https://stackoverflow.com/questions/66731036/unable-to-import-pyodbc-on-apple-silicon-symbol-not-found-sqlallochandle ## v1.4.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fb39952f..eca250d3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,7 +16,7 @@ pyenv activate dbt-synapse Install the development dependencies and pre-commit and get information about possible make commands: ```shell -make dev # if Windows, remove `pyodbc==4.0.39 --no-binary :all:` in dev_requirements.txt +make dev # for Mac users, add `pyodbc==4.0.39 --no-binary :all:` in dev_requirements.txt before running `make dev` make help ``` diff --git a/dev_requirements.txt b/dev_requirements.txt index 06071e8d..7e77f77b 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -5,7 +5,6 @@ pre-commit==2.21.0;python_version<"3.8" pre-commit==3.3.1;python_version>="3.8" pytest-dotenv==0.5.2 dbt-tests-adapter~=1.4.9 -pyodbc==4.0.39 --no-binary :all: # if Windows, remove aiohttp==3.8.3 azure-mgmt-synapse==2.0.0 flaky==3.7.0 From 0844ef1530ded3d28ef21bfa31e09668e56930a9 Mon Sep 17 00:00:00 2001 From: nszoni Date: Wed, 14 Feb 2024 12:09:02 +0100 Subject: [PATCH 37/41] remove redundant profile config --- tests/functional/adapter/test_copy_uppercase.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/functional/adapter/test_copy_uppercase.py b/tests/functional/adapter/test_copy_uppercase.py index 8717b752..1e425082 100644 --- a/tests/functional/adapter/test_copy_uppercase.py +++ b/tests/functional/adapter/test_copy_uppercase.py @@ -1,5 +1,4 @@ import pytest -from conftest import _profile_ci_azure_auto, _profile_user, _profile_user_azure from dbt.tests.adapter.simple_copy.fixtures import ( _MODELS__ADVANCED_INCREMENTAL, _MODELS__COMPOUND_SORT, @@ -16,17 +15,6 @@ from dbt.tests.util import check_relations_equal, run_dbt -def dbt_profile_target(request): - profile = request.config.getoption("--profile") - - if profile == "ci_azure_auto": - return _profile_ci_azure_auto() - if profile == "user": - return _profile_user() - if profile == "user_azure": - return _profile_user_azure() - - class TestSimpleCopyUppercase: @pytest.fixture(scope="class") def models(self): From 60d33ea42288e344c86f8462b0273f675cd1bad0 Mon Sep 17 00:00:00 2001 From: nszoni Date: Wed, 14 Feb 2024 16:48:23 +0100 Subject: [PATCH 38/41] use top instead of limit in dbt show and adapt tests --- dbt/include/synapse/macros/adapters/show.sql | 22 ++++++ tests/functional/adapter/test_dbt_show.py | 70 ++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 dbt/include/synapse/macros/adapters/show.sql create mode 100644 tests/functional/adapter/test_dbt_show.py diff --git a/dbt/include/synapse/macros/adapters/show.sql b/dbt/include/synapse/macros/adapters/show.sql new file mode 100644 index 00000000..4caa6eea --- /dev/null +++ b/dbt/include/synapse/macros/adapters/show.sql @@ -0,0 +1,22 @@ +{% macro get_show_sql(compiled_code, sql_header, limit) -%} + {%- if sql_header -%} + {{ sql_header }} + {%- endif -%} + {%- if limit is not none -%} + {{ get_limit_subquery_sql(compiled_code, limit) }} + {%- else -%} + {{ compiled_code }} + {%- endif -%} +{% endmacro %} + +{% macro get_limit_subquery_sql(sql, limit) %} + {{ adapter.dispatch('get_limit_subquery_sql', 'dbt')(sql, limit) }} +{% endmacro %} + +{# Synapse doesnt support ANSI LIMIT clause #} +{% macro synapse__get_limit_subquery_sql(sql, limit) %} + select top {{ limit }} * + from ( + {{ sql }} + ) as model_limit_subq +{% endmacro %} diff --git a/tests/functional/adapter/test_dbt_show.py b/tests/functional/adapter/test_dbt_show.py new file mode 100644 index 00000000..6cad1411 --- /dev/null +++ b/tests/functional/adapter/test_dbt_show.py @@ -0,0 +1,70 @@ +import pytest +from dbt.tests.adapter.dbt_show.test_dbt_show import ( + models__sample_model, + models__sql_header, + seeds__sample_seed, +) +from dbt.tests.util import run_dbt + +models__sample_model_a = """ +select + coalesce(sample_num, 0) + 10 as col_deci +from {{ ref('sample_model') }} +""" + +models__sample_model_b = """ +select + col_deci + 100 as col_hundo +from {{ ref('sample_model_a') }} +""" + + +# Synapse doesn't support ephemeral models so we need to alter the base tests +class BaseShowLimit: + @pytest.fixture(scope="class") + def models(self): + return { + "sample_model.sql": models__sample_model, + "sample_model_a.sql": models__sample_model_a, + } + + @pytest.fixture(scope="class") + def seeds(self): + return {"sample_seed.csv": seeds__sample_seed} + + @pytest.mark.parametrize( + "args,expected", + [ + ([], 5), # default limit + (["--limit", 3], 3), # fetch 3 rows + (["--limit", -1], 7), # fetch all rows + ], + ) + def test_limit(self, project, args, expected): + run_dbt(["build"]) + dbt_args = ["show", "--inline", models__sample_model_b, *args] + results = run_dbt(dbt_args) + assert len(results.results[0].agate_table) == expected + # ensure limit was injected in compiled_code when limit specified in command args + limit = results.args.get("limit") + if limit > 0: + assert f"top {limit}" in results.results[0].node.compiled_code + + +class BaseShowSqlHeader: + @pytest.fixture(scope="class") + def models(self): + return { + "sql_header.sql": models__sql_header, + } + + def test_sql_header(self, project): + run_dbt(["show", "--select", "sql_header", "--vars", "timezone: Asia/Kolkata"]) + + +class TestShowSqlHeaderSynapse(BaseShowSqlHeader): + pass + + +class TestShowLimitSynapse(BaseShowLimit): + pass From b8792d9aee02e0ad5e90d8ec9f028a2f1b4c1211 Mon Sep 17 00:00:00 2001 From: "Arthur Chionh (dataroots)" <90323829+arthurcht@users.noreply.github.com> Date: Mon, 19 Feb 2024 20:50:19 +0100 Subject: [PATCH 39/41] add permissions block in workflow .yml --- .github/workflows/integration-tests-azure.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 1e3e76b4..8e7e00c7 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -10,6 +10,15 @@ on: # yamllint disable-line rule:truthy - master jobs: + build-me: + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + deployments: read + packages: none + pull-requests: write + security-events: write integration-tests-azure: name: Integration tests on Azure strategy: From 151327467ad8dc1bd1cb36c6501c9223132c63d5 Mon Sep 17 00:00:00 2001 From: "Arthur Chionh (dataroots)" <90323829+arthurcht@users.noreply.github.com> Date: Mon, 19 Feb 2024 20:53:03 +0100 Subject: [PATCH 40/41] update permissions block in workflow .yml --- .github/workflows/integration-tests-azure.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 8e7e00c7..2decb350 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -10,7 +10,12 @@ on: # yamllint disable-line rule:truthy - master jobs: - build-me: + integration-tests-azure: + name: Integration tests on Azure + strategy: + matrix: + python_version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + msodbc_version: ["17", "18"] runs-on: ubuntu-latest permissions: actions: read @@ -19,13 +24,6 @@ jobs: packages: none pull-requests: write security-events: write - integration-tests-azure: - name: Integration tests on Azure - strategy: - matrix: - python_version: ["3.7", "3.8", "3.9", "3.10", "3.11"] - msodbc_version: ["17", "18"] - runs-on: ubuntu-latest container: image: ghcr.io/dbt-msft/dbt-sqlserver:CI-${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }} steps: From 9a58d9cb9ebbf23cff688cd837a40ae24d0b1e21 Mon Sep 17 00:00:00 2001 From: artc95 Date: Tue, 20 Feb 2024 09:29:16 +0100 Subject: [PATCH 41/41] commit datarootsio/dependency_dbtsqlserver_to_dbtfabric --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e1b2e67..082e56f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,9 @@ ## v1.4.1rc1 #### Under the hood -* Switch dependency from dbt-sqlserver to dbt-fabric (per https://github.com/dbt-msft/dbt-sqlserver/issues/441) +* Switch dependency from dbt-sqlserver to dbt-fabric [#441](https://github.com/dbt-msft/dbt-sqlserver/issues/441) * for Mac users, before running `make dev`, add `pyodbc==4.0.39 --no-binary :all:` in dev_requirements.txt - * about pyodbc "Symbol not found: _SQLAllocHandle" error https://stackoverflow.com/questions/66731036/unable-to-import-pyodbc-on-apple-silicon-symbol-not-found-sqlallochandle + * [Stackoverflow](https://stackoverflow.com/questions/66731036/unable-to-import-pyodbc-on-apple-silicon-symbol-not-found-sqlallochandle) about pyodbc "Symbol not found: _SQLAllocHandle" error ## v1.4.0