Feat(Realtime) - Realtime improvements

.gitattributes

@@ -1,7 +1,10 @@
 # See https://git-scm.com/docs/gitattributes for more about git attribute files.
 
 # Mark the database schema as having been generated.
-db/schema.rb linguist-generated
+db/structure.sql linguist-generated
+db/events_structure.sql linguist-generated
+db/clickhouse_structure.sql linguist-generated
+
 schema.json linguist-generated
 schema.graphql linguist-generated
 

.github/workflows/migrations-test.yml

@@ -53,13 +53,11 @@ jobs:
       - name: Generate RSA keys
         run: ./scripts/generate.rsa.sh
       - name: Move db schema for comparison
-        run: mv db/schema.rb db/schema-before-dump.rb
+        run: mv db/structure.sql db/structure-before-dump.sql
       - name: Perform Postgres database migrations
         run: bin/rails db:migrate:primary
       - name: dump schema
         run: LAGO_DISABLE_SCHEMA_DUMP="" bin/rails db:schema:dump:primary
-      - name: Ensure no changes to schema.rb
-        run: diff db/schema.rb db/schema-before-dump.rb
       - name: Ensure annotations are up to date
         run: bundle exec annotate --frozen
       - name: Perform Clickhouse database migrations

.github/workflows/pronto.yml

@@ -20,9 +20,9 @@ jobs:
           bundle install
       - name: Run Rubocop
         # We must pass the list of files because for now, rubocop on the entire project throws too many errors.
-        # We exclude the db/schema.rb file explicitly because passing a list of files will override the `AllCops.Exclude` config in .rubocop.yml
+        # We exclude the db/structure.sql file explicitly because passing a list of files will override the `AllCops.Exclude` config in .rubocop.yml
         run: |
-          FILES=$(git diff --diff-filter=d --name-only origin/${{ github.base_ref }}...HEAD -- '*.rb' ':!db/*schema.rb')
+          FILES=$(git diff --diff-filter=d --name-only origin/${{ github.base_ref }}...HEAD -- '*.rb' ':!db/*structure.sql')
           if [ -z "$FILES" ]; then
             echo "No Ruby files to lint"
             exit 0

.github/workflows/spec.yml

@@ -61,9 +61,7 @@ jobs:
         run: ./scripts/generate.rsa.sh
       - name: Set up Postgres database schema
         run: bin/rails db:schema:load:primary
-      - name: Set up Clickhouse database schema
-        run: bin/rails db:migrate:clickhouse
-      - name: Dump Clickhouse database schema
-        run: bin/rails db:schema:dump:clickhouse
+      - name: Set up Clickhouse
+        run: bin/rails db:migrate:clickhouse &&  bin/rails db:schema:dump:clickhouse
       - name: Run tests
         run: bundle exec rspec

.gitignore

@@ -22,8 +22,6 @@
 /tmp/storage/*
 !/tmp/storage/
 !/tmp/storage/.keep
-db/events_schema.rb
-db/clickhouse_schema.rb
 /config/master.key
 
 .env

.rubocop.yml

@@ -17,8 +17,6 @@ AllCops:
   DisplayStyleGuide: true
   Exclude:
     - 'bin/**/*'
-    - 'db/schema.rb'
-    - 'db/*_schema.rb'
 
 # TODO: Enable when we have time to fix all the offenses
 Style/StringLiterals:

Dockerfile.dev

@@ -3,7 +3,7 @@ FROM ruby:3.3.6-slim
 WORKDIR /app
 
 RUN apt update -qq && \
-  apt install nodejs build-essential curl git pkg-config libpq-dev libclang-dev -y && \
+  apt install nodejs build-essential postgresql-client curl git pkg-config libpq-dev libclang-dev -y && \
   curl https://sh.rustup.rs -sSf | bash -s -- -y
 
 ENV BUNDLER_VERSION='2.5.5'

Gemfile.lock

@@ -166,7 +166,7 @@ GEM
       activesupport (>= 3.0.0)
       uniform_notifier (~> 1.11)
     byebug (11.1.3)
-    clickhouse-activerecord (1.2.0)
+    clickhouse-activerecord (1.2.1)
       activerecord (~> 7.1)
       bundler (>= 1.13.4)
     clockwork (3.0.2)
@@ -386,7 +386,7 @@ GEM
     method_source (1.1.0)
     mini_mime (1.1.5)
     mini_portile2 (2.8.7)
-    minitest (5.25.1)
+    minitest (5.25.2)
     monetize (1.13.0)
       money (~> 6.12)
     money (6.19.0)

app/models/subscription_event_trigger.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class SubscriptionEventTrigger < ApplicationRecord
+  validates :organization_id, :external_subscription_id, :created_at, presence: true
+
+  scope :ordered, -> { order(created_at: :desc) }
+
+  def self.trigger(organization_id:, external_subscription_id:)
+    connection.select_all sanitize_sql_array(["call trigger_subscription_update(?,?, null)", organization_id, external_subscription_id])
+  end
+
+  def self.take
+    candidate = SubscriptionEventTrigger.ordered.limit(1)
+  end
+end
+
+# == Schema Information
+#
+# Table name: subscription_event_triggers
+#
+#  id                       :uuid             not null, primary key
+#  start_processing_at      :datetime
+#  created_at               :datetime         not null
+#  external_subscription_id :string           not null
+#  organization_id          :uuid             not null
+#
+# Indexes
+#
+#  idx_on_external_subscription_id_organization_id_40aa74e2eb      (external_subscription_id,organization_id) UNIQUE WHERE (start_processing_at IS NULL)
+#  idx_on_start_processing_at_external_subscription_id_31b81116ce  (start_processing_at,external_subscription_id,organization_id) UNIQUE
+#

config/application.rb

@@ -23,6 +23,9 @@ class Application < Rails::Application
     ]
 
     config.api_only = true
+
+    config.active_record.schema_format = :sql
+    ActiveRecord::Tasks::DatabaseTasks.structure_dump_flags = ['--clean', '--if-exists']
     config.active_job.queue_adapter = :sidekiq
 
     # Configuration for active record encryption

config/database.yml

@@ -9,7 +9,6 @@ development:
     password: changeme
     database: lago
     port: 5432
-    schema_search_path: 'public'
   events:
     <<: *default
     host: db
@@ -32,7 +31,7 @@ test:
   primary:
     <<: *default
     url: <%= ENV['DATABASE_TEST_URL'].presence || ENV['DATABASE_URL'] %>
-    schema_dump: <% if ENV['LAGO_DISABLE_SCHEMA_DUMP'].present? %> false <% else %> schema.rb <% end %>
+    schema_dump: <% if ENV['LAGO_DISABLE_SCHEMA_DUMP'].present? %> false <% else %> structure.sql <% end %>
   events:
     <<: *default
     url: <%= ENV['DATABASE_TEST_URL'].presence || ENV['DATABASE_URL'] %>
@@ -45,6 +44,7 @@ test:
     username: <%= ENV.fetch('LAGO_CLICKHOUSE_USERNAME', 'default') %>
     password: <%= ENV.fetch('LAGO_CLICKHOUSE_PASSWORD', 'default') %>
     migrations_paths: db/clickhouse_migrate
+    schema_format: 'ruby'
     debug: true
     database_tasks: <% if ENV['LAGO_CLICKHOUSE_MIGRATIONS_ENABLED'].present? %> true <% else %> false <% end %>
     schema_dump: <% if ENV['LAGO_DISABLE_SCHEMA_DUMP'].present? %> false <% else %> clickhouse_schema.rb <% end %>