Merge pull request #468 from Multiwoven/cherry-pick-ce-commit-01793ba…

…b54f352d0998eb92e3be3549114ef70d0

feat(CE): added dynamic_sql type to model

server/app/contracts/model_contracts.rb

@@ -42,8 +42,20 @@ class Create < Dry::Validation::Contract
     end
 
     rule(model: :configuration) do
-      if values[:model][:configuration].blank? && (values[:model][:query_type] == "ai_ml")
-        key.failure("Configuration is required for this query type")
+      query_type = values[:model][:query_type]
+
+      if %w[dynamic_sql ai_ml].include? query_type
+        if value.blank?
+          key.failure("Configuration is required for this query type")
+        elsif query_type == "ai_ml"
+          key.failure("Config must contain harvester details") unless value.key?("harvesters")
+        else
+          key.failure("Config must contain harvester & json_schema") unless %w[harvesters
+                                                                               json_schema].any? do |k|
+                                                                              value.key?(k)
+                                                                            end
+
+        end
       end
     end
   end

server/app/models/model.rb

@@ -15,26 +15,35 @@
 #  updated_at   :datetime         not null
 #
 class Model < ApplicationRecord
+  AI_ML_CONFIG_JSON_SCHEMA = Rails.root.join("app/models/schema_validations/models/configuration_aiml.json")
+  DYNAMIC_SQL_CONFIG_JSON_SCHEMA = Rails.root.join(
+    "app/models/schema_validations/models/configuration_dynamic_sql.json"
+  )
+
   validates :workspace_id, presence: true
   validates :connector_id, presence: true
   validates :name, presence: true
 
-  enum :query_type, %i[raw_sql dbt soql table_selector ai_ml]
+  enum :query_type, %i[raw_sql dbt soql table_selector ai_ml dynamic_sql]
 
   validates :query, presence: true, if: :requires_query?
   # Havesting configuration
   validates :configuration, presence: true, if: :requires_configuration?
+  validates :configuration, presence: true, json: { schema: lambda {
+                                                              configuration_schema_validation
+                                                            } }, if: :requires_configuration?
 
   belongs_to :workspace
   belongs_to :connector
 
   has_many :syncs, dependent: :destroy
   has_many :visual_components, dependent: :destroy
 
-  scope :data, -> { where(query_type: %i[raw_sql dbt soql table_selector]) }
+  scope :data, -> { where(query_type: %i[raw_sql dbt soql table_selector dynamic_sql]) }
   scope :ai_ml, -> { where(query_type: :ai_ml) }
 
   default_scope { order(updated_at: :desc) }
+
   def to_protocol
     Multiwoven::Integrations::Protocol::Model.new(
       name:,
@@ -49,6 +58,16 @@ def requires_query?
   end
 
   def requires_configuration?
-    %w[ai_ml].include?(query_type)
+    %w[ai_ml dynamic_sql].include?(query_type)
+  end
+
+  private
+
+  def configuration_schema_validation
+    if ai_ml?
+      AI_ML_CONFIG_JSON_SCHEMA
+    elsif dynamic_sql?
+      DYNAMIC_SQL_CONFIG_JSON_SCHEMA
+    end
   end
 end

server/app/models/schema_validations/models/configuration_aiml.json

@@ -0,0 +1,9 @@
+{
+  "type": "object",
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "properties": {
+    "harvesters": { "type": "array" },
+    "json_schema": { "type": "array" }
+  },
+  "required": ["harvesters"]
+}

server/app/models/schema_validations/models/configuration_dynamic_sql.json

@@ -0,0 +1,9 @@
+{
+  "type": "object",
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "properties": {
+    "harvesters": { "type": "array" },
+    "json_schema": { "type": "array" }
+  },
+  "required": ["harvesters", "json_schema"]
+}

server/spec/contracts/model_contracts_spec.rb

@@ -56,9 +56,43 @@
         }
       end
 
+      let(:valid_input_dynamic_sql) do
+        {
+          model: {
+            connector_id: 1,
+            name: "Model Name",
+            query: "SELECT * FROM table;",
+            query_type: "dynamic_sql",
+            primary_key: "id",
+            configuration: { "json_schema" => [], "harvesters" => [] }
+          }
+        }
+      end
+
+      let(:valid_input_ai_ml) do
+        {
+          model: {
+            connector_id: 1,
+            name: "Model Name",
+            query: "SELECT * FROM table;",
+            query_type: "dynamic_sql",
+            primary_key: "id",
+            configuration: { "harvesters" => [] }
+          }
+        }
+      end
+
       it "passes validation" do
         expect(contract.call(valid_inputs)).to be_success
       end
+
+      it "passes validation for dynamic_sql query_type" do
+        expect(contract.call(valid_input_dynamic_sql)).to be_success
+      end
+
+      it "passes validation for ai_ml query_type" do
+        expect(contract.call(valid_input_ai_ml)).to be_success
+      end
     end
 
     context "with missing query for query_type requiring it" do
@@ -118,7 +152,7 @@
       end
     end
 
-    context "with missing configuration for ai_ml query_type" do
+    context "with missing or invalid configuration for ai_ml query_type" do
       let(:invalid_inputs) do
         {
           model: {
@@ -130,10 +164,62 @@
         }
       end
 
+      let(:invalid_configuration) do
+        {
+          model: {
+            connector_id: 1,
+            name: "Model Name",
+            query_type: "ai_ml",
+            primary_key: "id",
+            configuration: { "test" => "new" }
+          }
+        }
+      end
+
       it "fails validation due to missing configuration" do
         result = contract.call(invalid_inputs)
         expect(result.errors[:model][:configuration]).to include("Configuration is required for this query type")
       end
+
+      it "fails validation due to invalid configuration" do
+        result = contract.call(invalid_configuration)
+        expect(result.errors[:model][:configuration]).to include("Config must contain harvester details")
+      end
+    end
+
+    context "with missing or invalid configuration for dynamic_sql query_type" do
+      let(:invalid_inputs) do
+        {
+          model: {
+            connector_id: 1,
+            name: "Model Name",
+            query_type: "dynamic_sql",
+            primary_key: "id"
+          }
+        }
+      end
+
+      let(:invalid_configuration) do
+        {
+          model: {
+            connector_id: 1,
+            name: "Model Name",
+            query_type: "dynamic_sql",
+            primary_key: "id",
+            configuration: { "test" => "new" }
+          }
+        }
+      end
+
+      it "fails validation due to missing configuration" do
+        result = contract.call(invalid_inputs)
+        expect(result.errors[:model][:configuration]).to include("Configuration is required for this query type")
+      end
+
+      it "fails validation due to invalid configuration" do
+        result = contract.call(invalid_configuration)
+        expect(result.errors[:model][:configuration]).to include("Config must contain harvester & json_schema")
+      end
     end
   end
 

server/spec/models/model_spec.rb

@@ -53,7 +53,8 @@
           name: "test_model",
           query_type: :ai_ml, connector_id: source.id,
           workspace_id: source.workspace_id,
-          configuration: { "field1" => "value1" }
+          configuration: { harvesters:
+                     [{ "value" => "dynamic test", "method" => "dom", "selector" => "dom_id", "preprocess" => "" }] }
         )
         model.query = nil
         expect(model).to be_valid
@@ -62,13 +63,64 @@
 
     context "when query_type requires configuration" do
       it "validates presence of configuration" do
-        model = Model.new(
+        ai_ml_model = Model.new(
           name: "test_model",
           query_type: :ai_ml, connector_id: source.id,
           workspace_id: source.workspace_id
         )
-        model.configuration = nil
-        expect(model).not_to be_valid
+        dynamic_sql_model = Model.new(
+          name: "test_model",
+          query_type: :dynamic_sql, connector_id: source.id,
+          workspace_id: source.workspace_id
+        )
+        ai_ml_model.configuration = nil
+        dynamic_sql_model.configuration = nil
+        expect(ai_ml_model).not_to be_valid
+        expect(dynamic_sql_model).not_to be_valid
+      end
+
+      context "validates json schema of configuration" do
+        context "validates json schema of ai_ml models" do
+          it "is does returns invalid for ai ml models without valid configuration" do
+            ai_ml_model = Model.new(
+              name: "test_model",
+              query_type: :ai_ml, connector_id: source.id,
+              workspace_id: source.workspace_id,
+              configuration: { "harvesters": { "wrong": "format" } }
+            )
+            expect(ai_ml_model).not_to be_valid
+          end
+          it "is does returns valid for ai ml models with valid configuration" do
+            ai_ml_model = Model.new(
+              name: "test_model",
+              query_type: :ai_ml, connector_id: source.id,
+              workspace_id: source.workspace_id,
+              configuration: { "harvesters": [] }
+            )
+            expect(ai_ml_model).to be_valid
+          end
+        end
+
+        context "validates json schema of dynamic_sql models" do
+          it "is does returns invalid for dyanmic models without valid configuration" do
+            dynamic_sql_model = Model.new(
+              name: "test_model",
+              query_type: :dynamic_sql, connector_id: source.id,
+              workspace_id: source.workspace_id,
+              configuration: { "harvesters": { "wrong": "format" } }
+            )
+            expect(dynamic_sql_model).not_to be_valid
+          end
+          it "is does returns valid for ai ml models with valid configuration" do
+            dynamic_sql_model = Model.new(
+              name: "test_model",
+              query_type: :dynamic_sql, connector_id: source.id,
+              workspace_id: source.workspace_id,
+              configuration: { "harvesters": [], "json_schema": [] }
+            )
+            expect(dynamic_sql_model).to be_valid
+          end
+        end
       end
     end
 
@@ -128,7 +180,8 @@
 
   describe "query_type" do
     it "defines query_type enum with specified values" do
-      expect(Model.query_types).to eq({ "raw_sql" => 0, "dbt" => 1, "soql" => 2, "table_selector" => 3, "ai_ml" => 4 })
+      expect(Model.query_types).to eq({ "raw_sql" => 0, "dbt" => 1, "soql" => 2, "table_selector" => 3, "ai_ml" => 4,
+                                        "dynamic_sql" => 5 })
     end
   end
 
@@ -138,12 +191,48 @@
     let!(:dbt_model) { create(:model, query_type: :dbt, connector: source) }
     let!(:soql_model) { create(:model, query_type: :soql, connector: source) }
     let!(:table_selector_model) { create(:model, query_type: :table_selector, connector: source) }
-    let!(:ai_ml_model) { create(:model, query_type: :ai_ml, connector: source, configuration: { test: "value" }) }
+    let!(:ai_ml_model) do
+      create(:model, query_type: :ai_ml, connector: source,
+                     configuration: { harvesters:
+                     [{ "value" => "dynamic test", "method" => "dom", "selector" => "dom_id", "preprocess" => "" }] })
+    end
+    let!(:dynamic_sql_model) do
+      create(:model, query_type: :dynamic_sql, connector: source,
+                     configuration:
+                      {
+                        "harvesters": [
+                          {
+                            "value": "dynamic test",
+                            "method": "dom",
+                            "selector": "dom_id",
+                            "preprocess": ""
+                          }
+                        ],
+                        "json_schema": [
+                          {
+                            "input": [
+                              {
+                                "name": "risk_level",
+                                "type": "string",
+                                "value": "",
+                                "value_type": "dynamic"
+                              }
+                            ],
+                            "output": [
+                              {
+                                "name": "data.col0.calculated_risk",
+                                "type": "string"
+                              }
+                            ]
+                          }
+                        ]
+                      })
+    end
 
     describe ".data" do
       it "returns models with query_type in [raw_sql, dbt, soql, table_selector]" do
         data_models = Model.data
-        expect(data_models).to include(raw_sql_model, dbt_model, soql_model, table_selector_model)
+        expect(data_models).to include(raw_sql_model, dbt_model, soql_model, table_selector_model, dynamic_sql_model)
         expect(data_models).not_to include(ai_ml_model)
       end
     end