From f21bf67944654fba01811616a157e80d6bb01929 Mon Sep 17 00:00:00 2001 From: afthab vp Date: Tue, 9 Apr 2024 16:06:30 +0530 Subject: [PATCH] fix: add connector_query_type in spec (#23) --- integrations/Gemfile.lock | 2 +- .../integrations/protocol/protocol.rb | 2 +- .../lib/multiwoven/integrations/rollout.rb | 2 +- .../source/bigquery/config/spec.json | 1 + .../source/databricks/config/spec.json | 1 + .../source/postgresql/config/spec.json | 1 + .../source/redshift/config/spec.json | 1 + .../config/spec.json | 1 + .../source/snowflake/config/spec.json | 1 + .../integrations/protocol/protocol_spec.rb | 35 ++++++++----------- 10 files changed, 24 insertions(+), 23 deletions(-) diff --git a/integrations/Gemfile.lock b/integrations/Gemfile.lock index 352e058b..3371a166 100644 --- a/integrations/Gemfile.lock +++ b/integrations/Gemfile.lock @@ -7,7 +7,7 @@ GIT PATH remote: . specs: - multiwoven-integrations (0.1.54) + multiwoven-integrations (0.1.55) activesupport async-websocket csv diff --git a/integrations/lib/multiwoven/integrations/protocol/protocol.rb b/integrations/lib/multiwoven/integrations/protocol/protocol.rb index 072baf4a..95c554cc 100644 --- a/integrations/lib/multiwoven/integrations/protocol/protocol.rb +++ b/integrations/lib/multiwoven/integrations/protocol/protocol.rb @@ -57,6 +57,7 @@ class ConnectorSpecification < ProtocolModel attribute :supports_dbt, Types::Bool.default(false) attribute :stream_type, StreamType attribute? :supported_destination_sync_modes, Types::Array.of(DestinationSyncMode).optional + attribute? :connector_query_type, ConnectorQueryType def to_multiwoven_message MultiwovenMessage.new( @@ -70,7 +71,6 @@ class Connector < ProtocolModel attribute :name, Types::String attribute :type, ConnectorType attribute :connection_specification, Types::Hash - attribute :query_type, ConnectorQueryType.optional.default("raw_sql") end class LogMessage < ProtocolModel diff --git a/integrations/lib/multiwoven/integrations/rollout.rb b/integrations/lib/multiwoven/integrations/rollout.rb index 76acec0e..c5409f15 100644 --- a/integrations/lib/multiwoven/integrations/rollout.rb +++ b/integrations/lib/multiwoven/integrations/rollout.rb @@ -2,7 +2,7 @@ module Multiwoven module Integrations - VERSION = "0.1.54" + VERSION = "0.1.55" ENABLED_SOURCES = %w[ Snowflake diff --git a/integrations/lib/multiwoven/integrations/source/bigquery/config/spec.json b/integrations/lib/multiwoven/integrations/source/bigquery/config/spec.json index ad2444ed..8429e773 100644 --- a/integrations/lib/multiwoven/integrations/source/bigquery/config/spec.json +++ b/integrations/lib/multiwoven/integrations/source/bigquery/config/spec.json @@ -1,6 +1,7 @@ { "documentation_url": "https://docs.multiwoven.com/integrations/sources/bigquery", "stream_type": "dynamic", + "connector_query_type": "raw_sql", "connection_specification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Google BigQuery", diff --git a/integrations/lib/multiwoven/integrations/source/databricks/config/spec.json b/integrations/lib/multiwoven/integrations/source/databricks/config/spec.json index f5e03fa6..e446ffe3 100644 --- a/integrations/lib/multiwoven/integrations/source/databricks/config/spec.json +++ b/integrations/lib/multiwoven/integrations/source/databricks/config/spec.json @@ -1,6 +1,7 @@ { "documentation_url": "https://docs.multiwoven.com/integrations/sources/databricks", "stream_type": "dynamic", + "connector_query_type": "raw_sql", "connection_specification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Databricks", diff --git a/integrations/lib/multiwoven/integrations/source/postgresql/config/spec.json b/integrations/lib/multiwoven/integrations/source/postgresql/config/spec.json index 7ce099e6..b7e9a64a 100644 --- a/integrations/lib/multiwoven/integrations/source/postgresql/config/spec.json +++ b/integrations/lib/multiwoven/integrations/source/postgresql/config/spec.json @@ -1,6 +1,7 @@ { "documentation_url": "https://docs.multiwoven.com/integrations/sources/postgresql", "stream_type": "dynamic", + "connector_query_type": "raw_sql", "connection_specification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Postgresql", diff --git a/integrations/lib/multiwoven/integrations/source/redshift/config/spec.json b/integrations/lib/multiwoven/integrations/source/redshift/config/spec.json index e9cbb9f3..d9cbe434 100644 --- a/integrations/lib/multiwoven/integrations/source/redshift/config/spec.json +++ b/integrations/lib/multiwoven/integrations/source/redshift/config/spec.json @@ -1,6 +1,7 @@ { "documentation_url": "https://docs.multiwoven.com/integrations/sources/redshift", "stream_type": "dynamic", + "connector_query_type": "raw_sql", "connection_specification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Amazon Redshift", diff --git a/integrations/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/config/spec.json b/integrations/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/config/spec.json index 8d407f1c..5607a730 100644 --- a/integrations/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/config/spec.json +++ b/integrations/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/config/spec.json @@ -1,6 +1,7 @@ { "documentation_url": "https://docs.multiwoven.com/destinations/retail/salesforce_consumer_goods_cloud", "stream_type": "static", + "connector_query_type": "soql", "connection_specification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Salesforce Consumer Goods Cloud", diff --git a/integrations/lib/multiwoven/integrations/source/snowflake/config/spec.json b/integrations/lib/multiwoven/integrations/source/snowflake/config/spec.json index 2d9b23d5..6b9f282d 100644 --- a/integrations/lib/multiwoven/integrations/source/snowflake/config/spec.json +++ b/integrations/lib/multiwoven/integrations/source/snowflake/config/spec.json @@ -1,6 +1,7 @@ { "documentation_url": "https://docs.multiwoven.com/integrations/sources/snowflake", "stream_type": "dynamic", + "connector_query_type": "raw_sql", "connection_specification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Snowflake", diff --git a/integrations/spec/multiwoven/integrations/protocol/protocol_spec.rb b/integrations/spec/multiwoven/integrations/protocol/protocol_spec.rb index 8ad08e53..a14cf9ea 100644 --- a/integrations/spec/multiwoven/integrations/protocol/protocol_spec.rb +++ b/integrations/spec/multiwoven/integrations/protocol/protocol_spec.rb @@ -38,22 +38,34 @@ module Integrations::Protocol "supports_normalization" => true, "supports_dbt" => true, "supported_destination_sync_modes" => ["insert"] - }.to_json + } describe ".from_json" do it "creates an instance from JSON" do - instance = ConnectorSpecification.from_json(json_data) + instance = ConnectorSpecification.from_json(json_data.to_json) + expect(instance).to be_a(ConnectorSpecification) + expect(instance.connection_specification).to eq(key: "value") + expect(instance.supports_normalization).to eq(true) + expect(instance.supports_dbt).to eq(true) + expect(instance.supported_destination_sync_modes).to eq(["insert"]) + expect(instance.connector_query_type).to eq(nil) + end + + it "creates an instance from JSON connector_query_type soql" do + json_data[:connector_query_type] = "soql" + instance = ConnectorSpecification.from_json(json_data.to_json) expect(instance).to be_a(ConnectorSpecification) expect(instance.connection_specification).to eq(key: "value") expect(instance.supports_normalization).to eq(true) expect(instance.supports_dbt).to eq(true) expect(instance.supported_destination_sync_modes).to eq(["insert"]) + expect(instance.connector_query_type).to eq("soql") end end describe "#to_multiwoven_message" do it "converts to a MultiwovenMessage" do - connector_spec = described_class.from_json(json_data) + connector_spec = described_class.from_json(json_data.to_json) multiwoven_message = connector_spec.to_multiwoven_message expect(multiwoven_message).to be_a(Multiwoven::Integrations::Protocol::MultiwovenMessage) @@ -359,23 +371,6 @@ module Integrations::Protocol expect(connector.connection_specification).to eq(key: "value") end end - - context "connector_query_type validations" do - it "has a connector_query_type 'sql'" do - connector = Connector.new(name: "Test", type: "source", query_type: "soql", connection_specification: {}) - expect(ModelQueryType.values).to include(connector.query_type) - end - - it "has a connector_query_type 'soql'" do - connector = Connector.new(name: "Test", type: "destination", query_type: "soql", connection_specification: {}) - expect(ModelQueryType.values).to include(connector.query_type) - end - - it "has a query_type default raw_sql " do - connector = Connector.new(name: "Test", type: "destination", connection_specification: {}) - expect(connector.query_type).to eq("raw_sql") - end - end end RSpec.describe Multiwoven::Integrations::Protocol::ControlMessage do