handle soft deletion for schema records that not present on the snapshot

app/services/schema_snapshot_service.rb

@@ -38,13 +38,17 @@ def schema_changed?
 
   private
 
-  # TODO: filter out Views (table_type = "BASE TABLE")
+
+  # filter out views
   def current_schema_data
     @current_schema_data ||= ActiveRecord::Base.connection.select_all(<<-SQL).to_a
-      SELECT table_name, column_name, data_type, is_nullable
-      FROM information_schema.columns
-      WHERE table_schema = '#{@schema_name}'
-      ORDER BY table_name, ordinal_position;
+      SELECT c.table_name, c.column_name, c.data_type, c.is_nullable
+      FROM information_schema.columns c
+      JOIN information_schema.tables t 
+        ON c.table_name = t.table_name
+        AND c.table_schema = t.table_schema
+      WHERE c.table_schema = '#{@schema_name}' AND t.table_type = 'BASE TABLE'
+      ORDER BY c.table_name, c.ordinal_position;
     SQL
   end
-end
+end

app/services/schema_sync_service.rb

@@ -3,18 +3,34 @@ def initialize(schema_name)
     @schema_name = schema_name
   end
 
-  def sync_schema_with_documentation
+  def sync_schema_with_latest_snapshot
     schema_snapshot = fetch_schema_snapshot
 
-    schema_snapshot.each do |item|
-      table_name = item['table_name']
-      column_name = item['column_name']
-      doc_record = find_or_initialize_documentation(table_name, column_name)
-      doc_record.data_type = item['data_type'] # TODO: normalize data type
-      doc_record.save!
+    snapshot_hash = prepare_snapshot(schema_snapshot)
+
+    schema_records.each do |record|
+      key = [record.table_name, record.column_name]
+      if snapshot_hash.key?(key)
+        record.update!(
+          data_type: snapshot_hash[key]["data_type"],
+          active: true
+        )
+      snapshot_hash.delete(key)
+      else
+        puts "Marking #{record.table_name}.#{record.column_name} as inactive"
+        record.update!(active: false) # if not a part of the snapshot, mark as inactive
+      end
     end
 
-    # TODO: add inactive records that not present in the shema snapshot
+    snapshot_hash.each_value do |item|
+      Support::CtgovSchema.create!(
+        table_name: item["table_name"],
+        column_name: item["column_name"],
+        data_type: item["data_type"],
+        active: true
+      )
+    end
+    # TODO: add is nullable?
   end
 
   private
@@ -24,7 +40,16 @@ def fetch_schema_snapshot
     Support::SchemaSnapshot.where(schema_name: @schema_name).order(created_at: :desc).first.snapshot
   end
 
-  def find_or_initialize_documentation(table_name, column_name)
-    Support::CtgovSchema.find_or_initialize_by(table_name: table_name, column_name: column_name)
+  def schema_records
+    # TODO: link passed schema to the model used to retrieve the schema values
+    Support::CtgovSchema.all # for not is okay since only one schema is used
+  end
+
+  # convert snapshot to hash for easy access by table_name and column_name
+  def prepare_snapshot(snapshot)
+    snapshot.each_with_object({}) do |item, hash|
+      key = [item["table_name"], item["column_name"]]
+      hash[key] = item
+    end
   end
 end