v1.5.0rc

CHANGELOG.md

@@ -1,4 +1,23 @@
 # Changelog
+## v.1.5.0rc1
+
+* Support for [dbt-core 1.5](https://github.com/dbt-labs/dbt-core/releases/tag/v1.5.0)
+  * Add support for model contracts by adapting `create_table_as` and `create_view_as` macros
+  * Define supported constraints in `CONSTRAINT_SUPPORT` Adapter class.
+  * Persist docs via [extended properties](https://github.com/dbt-msft/dbt-sqlserver/issues/134) is [not supported](https://learn.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-addextendedproperty-transact-sql?view=sql-server-ver16) in Synapse
+  * Add adapter tests zones
+    - caching
+    - column_types
+    - constraints
+    - hooks
+    - simple_copy
+
+## v1.4.1rc1
+
+#### Under the hood
+* Switch dependency from dbt-sqlserver to dbt-fabric (per https://github.com/dbt-msft/dbt-sqlserver/issues/441)
+  * if Windows, remove `pyodbc==4.0.39 --no-binary :all:` in dev_requirements.txt
+
 ## v1.4.0
 
 #### Features

CONTRIBUTING.md

@@ -16,7 +16,7 @@ pyenv activate dbt-synapse
 Install the development dependencies and pre-commit and get information about possible make commands:
 
 ```shell
-make dev
+make dev  # if Windows, remove `pyodbc==4.0.39 --no-binary :all:` in dev_requirements.txt
 make help
 ```
 

dbt/adapters/synapse/__init__.py

@@ -1,6 +1,7 @@
 from dbt.adapters.base import AdapterPlugin
 
 from dbt.adapters.synapse.synapse_adapter import SynapseAdapter
+from dbt.adapters.synapse.synapse_column import SynapseColumn
 from dbt.adapters.synapse.synapse_connection_manager import SynapseConnectionManager
 from dbt.adapters.synapse.synapse_credentials import SynapseCredentials
 from dbt.include import synapse
@@ -9,7 +10,13 @@
     adapter=SynapseAdapter,
     credentials=SynapseCredentials,
     include_path=synapse.PACKAGE_PATH,
-    dependencies=["sqlserver"],
+    dependencies=["fabric"],
 )
 
-__all__ = ["Plugin", "SynapseConnectionManager", "SynapseAdapter", "SynapseCredentials"]
+__all__ = [
+    "Plugin",
+    "SynapseConnectionManager",
+    "SynapseColumn",
+    "SynapseAdapter",
+    "SynapseCredentials",
+]

dbt/adapters/synapse/__version__.py

@@ -1 +1 @@
-version = "1.4.0"
+version = "1.5.0rc1"

dbt/adapters/synapse/synapse_adapter.py

@@ -1,7 +1,81 @@
-from dbt.adapters.sqlserver import SQLServerAdapter
+from enum import Enum
+from typing import Any, Dict, List, Optional
 
+from dbt.adapters.base.relation import BaseRelation
+from dbt.adapters.cache import _make_ref_key_dict
+from dbt.adapters.fabric import FabricAdapter
+from dbt.adapters.sql.impl import CREATE_SCHEMA_MACRO_NAME
+from dbt.contracts.graph.nodes import ColumnLevelConstraint, ConstraintType
+from dbt.events.functions import fire_event
+from dbt.events.types import SchemaCreation
+
+from dbt.adapters.synapse.synapse_column import SynapseColumn
 from dbt.adapters.synapse.synapse_connection_manager import SynapseConnectionManager
 
 
-class SynapseAdapter(SQLServerAdapter):
+class SynapseAdapter(FabricAdapter):
     ConnectionManager = SynapseConnectionManager
+    Column = SynapseColumn
+
+    def create_schema(self, relation: BaseRelation) -> None:
+        relation = relation.without_identifier()
+        fire_event(SchemaCreation(relation=_make_ref_key_dict(relation)))
+        macro_name = CREATE_SCHEMA_MACRO_NAME
+        kwargs = {
+            "relation": relation,
+        }
+
+        if self.config.credentials.schema_authorization:
+            kwargs["schema_authorization"] = self.config.credentials.schema_authorization
+            macro_name = "synapse__create_schema_with_authorization"
+
+        self.execute_macro(macro_name, kwargs=kwargs)
+        self.commit_if_has_connection()
+
+    class ConstraintSupport(str, Enum):
+        ENFORCED = "enforced"
+        NOT_ENFORCED = "not_enforced"
+        NOT_SUPPORTED = "not_supported"
+
+    # https://learn.microsoft.com/en-us/azure/synapse-analytics/sql-data-warehouse/sql-data-warehouse-table-constraints#table-constraints
+    CONSTRAINT_SUPPORT = {
+        ConstraintType.check: ConstraintSupport.NOT_SUPPORTED,  # no CHECK support for Synapse
+        ConstraintType.not_null: ConstraintSupport.ENFORCED,
+        ConstraintType.unique: ConstraintSupport.NOT_ENFORCED,
+        ConstraintType.primary_key: ConstraintSupport.NOT_ENFORCED,
+        ConstraintType.foreign_key: ConstraintSupport.NOT_SUPPORTED,  # no FK support for Synapse
+    }
+
+    @classmethod
+    def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> Optional[str]:
+        """Render the given constraint as DDL text.
+        Should be overriden by adapters which need custom constraint rendering."""
+        if constraint.type == ConstraintType.check and constraint.expression:
+            return f"check {constraint.expression}"
+        elif constraint.type == ConstraintType.not_null:
+            return "not null"
+        elif constraint.type == ConstraintType.unique:
+            return "unique NOT ENFORCED"
+        elif constraint.type == ConstraintType.primary_key:
+            return "primary key NONCLUSTERED NOT ENFORCED"
+        elif constraint.type == ConstraintType.foreign_key:
+            return "foreign key"
+        elif constraint.type == ConstraintType.custom and constraint.expression:
+            return constraint.expression
+        else:
+            return None
+
+    @classmethod
+    def render_raw_columns_constraints(cls, raw_columns: Dict[str, Dict[str, Any]]) -> List:
+        rendered_column_constraints = []
+
+        for v in raw_columns.values():
+            rendered_column_constraint = [f"{v['name']} {v['data_type']}"]
+            for con in v.get("constraints", None):
+                constraint = cls._parse_column_constraint(con)
+                c = cls.process_parsed_constraint(constraint, cls.render_column_constraint)
+                if c is not None:
+                    rendered_column_constraint.append(c)
+            rendered_column_constraints.append(" ".join(rendered_column_constraint))
+
+        return rendered_column_constraints

dbt/adapters/synapse/synapse_column.py

@@ -0,0 +1,16 @@
+from dbt.adapters.fabric import FabricColumn
+
+
+class SynapseColumn(FabricColumn):
+    # extending list of integer types for synapse
+    def is_integer(self) -> bool:
+        return self.dtype.lower() in [
+            # real types
+            "smallint",
+            "bigint",
+            "tinyint",
+            "serial",
+            "bigserial",
+            "int",
+            "bit",
+        ]

dbt/adapters/synapse/synapse_connection_manager.py

@@ -1,6 +1,6 @@
-from dbt.adapters.sqlserver import SQLServerConnectionManager
+from dbt.adapters.fabric import FabricConnectionManager
 
 
-class SynapseConnectionManager(SQLServerConnectionManager):
+class SynapseConnectionManager(FabricConnectionManager):
     TYPE = "synapse"
     TOKEN = None

dbt/adapters/synapse/synapse_credentials.py

@@ -1,10 +1,10 @@
 from dataclasses import dataclass
 
-from dbt.adapters.sqlserver import SQLServerCredentials
+from dbt.adapters.fabric import FabricCredentials
 
 
 @dataclass
-class SynapseCredentials(SQLServerCredentials):
+class SynapseCredentials(FabricCredentials):
     @property
     def type(self):
         return "synapse"

dbt/include/synapse/macros/adapters/indexes.sql

@@ -126,3 +126,35 @@ declare @drop_remaining_indexes_last nvarchar(max) = (
 {% macro create_nonclustered_index(columns, includes=False) %}
   {{ return(create_nonclustered_index(columns, includes=False)) }}
 {% endmacro %}
+
+
+{% macro drop_fk_indexes_on_table(relation) -%}
+  {% call statement('find_references', fetch_result=true) %}
+      SELECT  obj.name AS FK_NAME,
+      sch.name AS [schema_name],
+      tab1.name AS [table],
+      col1.name AS [column],
+      tab2.name AS [referenced_table],
+      col2.name AS [referenced_column]
+      FROM sys.foreign_key_columns fkc
+      INNER JOIN sys.objects obj
+          ON obj.object_id = fkc.constraint_object_id
+      INNER JOIN sys.tables tab1
+          ON tab1.object_id = fkc.parent_object_id
+      INNER JOIN sys.schemas sch
+          ON tab1.schema_id = sch.schema_id
+      INNER JOIN sys.columns col1
+          ON col1.column_id = parent_column_id AND col1.object_id = tab1.object_id
+      INNER JOIN sys.tables tab2
+          ON tab2.object_id = fkc.referenced_object_id
+      INNER JOIN sys.columns col2
+          ON col2.column_id = referenced_column_id AND col2.object_id = tab2.object_id
+      WHERE sch.name = '{{ relation.schema }}' and tab2.name = '{{ relation.identifier }}'
+  {% endcall %}
+      {% set references = load_result('find_references')['data'] %}
+      {% for reference in references -%}
+        {% call statement('main') -%}
+           alter table [{{reference[1]}}].[{{reference[2]}}] drop constraint [{{reference[0]}}]
+        {%- endcall %}
+      {% endfor %}
+{% endmacro %}

dbt/include/synapse/macros/adapters/persist_docs.sql

@@ -0,0 +1,4 @@
+{# Unfortunately adding docs via extended properties is not supported in Synapse only in SQLServer
+  https://github.com/dbt-msft/dbt-sqlserver/issues/134
+  https://learn.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-addextendedproperty-transact-sql?view=sql-server-ver16
+ #}

dbt/include/synapse/macros/adapters/relation.sql

@@ -20,7 +20,23 @@
 
 {% macro synapse__rename_relation(from_relation, to_relation) -%}
   {% call statement('rename_relation') -%}
+  -- drop all object types with to_relation.identifier name, to avoid error "new name already in use...duplicate...not permitted"
+  if object_id ('{{ to_relation.include(database=False) }}','V') is not null
+    begin
+    drop view {{ to_relation.include(database=False) }}
+    end
 
-    rename object {{ from_relation.include(database=False) }} to {{ to_relation.identifier }}
+  if object_id ('{{ to_relation.include(database=False) }}','U') is not null
+    begin
+    drop table {{ to_relation.include(database=False) }}
+    end
+
+  rename object {{ from_relation.include(database=False) }} to {{ to_relation.identifier }}
   {%- endcall %}
 {% endmacro %}
+
+{% macro synapse__truncate_relation(relation) %}
+    {% call statement('truncate_relation') -%}
+        truncate table {{ relation }}
+    {%- endcall %}
+{% endmacro %}

dbt/include/synapse/macros/adapters/schema.sql

@@ -6,3 +6,32 @@
     END
   {% endcall %}
 {% endmacro %}
+
+{% macro synapse__create_schema_with_authorization(relation, schema_authorization) -%}
+  {% call statement('create_schema') -%}
+    IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ relation.schema }}')
+    BEGIN
+    EXEC('CREATE SCHEMA [{{ relation.schema }}] AUTHORIZATION [{{ schema_authorization }}]')
+    END
+  {% endcall %}
+{% endmacro %}
+
+{% macro synapse__drop_schema(relation) -%}
+  {%- set relations_in_schema = list_relations_without_caching(relation) %}
+
+  {% for row in relations_in_schema %}
+    {%- set schema_relation = api.Relation.create(database=relation.database,
+                                               schema=relation.schema,
+                                               identifier=row[1],
+                                               type=row[3]
+                                               ) -%}
+    {% do drop_relation(schema_relation) %}
+  {%- endfor %}
+
+  {% call statement('drop_schema') -%}
+    IF EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ relation.without_identifier().schema }}')
+    BEGIN
+    EXEC('DROP SCHEMA [{{ relation.without_identifier().schema }}]')
+    END
+  {% endcall %}
+{% endmacro %}

dbt/include/synapse/macros/materializations/models/table/create_table_as.sql

@@ -10,16 +10,35 @@
 
    {{ synapse__drop_relation_script(relation) }}
 
-   EXEC('create view [{{ tmp_relation.schema }}].[{{ tmp_relation.identifier }}] as
-    {{ temp_view_sql }}
-    ');
-
-  CREATE TABLE {{ relation.include(database=False) }}
-    WITH(
-      DISTRIBUTION = {{dist}},
-      {{index}}
-      )
-    AS (SELECT * FROM [{{ tmp_relation.schema }}].[{{ tmp_relation.identifier }}])
+   {{ synapse__create_view_as(tmp_relation, sql) }}
+
+   {% set contract_config = config.get('contract') %}
+
+   {% if contract_config.enforced %}
+
+        {{exceptions.warn("Model contracts cannot be enforced by <adapter>!")}}
+
+        CREATE TABLE [{{relation.schema}}].[{{relation.identifier}}]
+        {{ synapse__build_columns_constraints(tmp_relation) }}
+        WITH(
+          DISTRIBUTION = {{dist}},
+          {{index}}
+        )
+        {{ get_assert_columns_equivalent(sql)  }}
+
+        {% set listColumns %}
+            {% for column in model['columns'] %}
+                {{ "["~column~"]" }}{{ ", " if not loop.last }}
+            {% endfor %}
+        {%endset%}
+        {{ synapse__build_model_constraints(relation) }}
+
+        INSERT INTO [{{relation.schema}}].[{{relation.identifier}}]
+        ({{listColumns}}) SELECT {{listColumns}} FROM [{{tmp_relation.schema}}].[{{tmp_relation.identifier}}]
+
+    {%- else %}
+        EXEC('CREATE TABLE [{{relation.database}}].[{{relation.schema}}].[{{relation.identifier}}]WITH(DISTRIBUTION = {{dist}},{{index}}) AS (SELECT * FROM [{{tmp_relation.database}}].[{{tmp_relation.schema}}].[{{tmp_relation.identifier}}]);');
+    {% endif %}
 
    {{ synapse__drop_relation_script(tmp_relation) }}
 

dbt/include/synapse/macros/materializations/models/table/create_table_constraints.sql

@@ -0,0 +1,17 @@
+{% macro synapse__build_columns_constraints(relation) %}
+  {# loop through user_provided_columns to create DDL with data types and constraints #}
+    {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%}
+    (
+      {% for c in raw_column_constraints -%}
+        {{ c }}{{ "," if not loop.last }}
+      {% endfor %}
+    )
+{% endmacro %}
+
+{% macro synapse__build_model_constraints(relation) %}
+  {# loop through user_provided_columns to create DDL with data types and constraints #}
+    {%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%}
+    {% for c in raw_model_constraints -%}
+        alter table {{ relation.include(database=False) }} {{c}};
+    {% endfor -%}
+{% endmacro %}

dbt/include/synapse/macros/materializations/models/view/create_view_as.sql

@@ -1,4 +1,15 @@
 {% macro synapse__create_view_as(relation, sql) -%}
-  create view {{ relation.include(database=False) }} as
-    {{ sql }}
+
+  {%- set temp_view_sql = sql.replace("'", "''") -%}
+
+  {% set contract_config = config.get('contract') %}
+
+    {{exceptions.warn("Model contracts cannot be enforced by <adapter>!")}}
+
+  {% if contract_config.enforced %}
+      {{ get_assert_columns_equivalent(sql) }}
+  {%- endif %}
+
+  EXEC('create view {{ relation.include(database=False) }} as {{ temp_view_sql }};');
+
 {% endmacro %}