From 3b928c9880b4302c8dd800afc20dac9e981cd71b Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sat, 8 Jun 2024 18:50:20 +0200 Subject: [PATCH 01/26] Dockerize, Postgresize, Redisize, and Sidekiqize Tap --- .github/workflows/ruby.yml | 33 ++-- .irbrc | 29 +++ .ruby-version | 2 +- .tool-versions | 2 +- Gemfile | 23 +-- Gemfile.lock | 184 +++++++++--------- Procfile.dev | 3 + README.md | 55 ++---- app/controllers/concerns/statistics.rb | 2 +- app/controllers/orders_controller.rb | 2 +- app/jobs/tab_api_job.rb | 10 +- app/models/order.rb | 6 +- bin/delayed_job | 5 - bin/dev | 10 + config/application.rb | 4 +- config/database.yml | 25 +-- config/deploy.rb | 23 +-- config/deploy/staging.rb | 7 - config/environments/development.rb | 1 - config/initializers/delayed_job_config.rb | 2 - config/initializers/sidekiq.rb | 3 + config/routes.rb | 6 + config/secrets.yml | 11 +- config/sidekiq.yml | 5 + ...20240608163258_remove_delayed_job_table.rb | 5 + db/schema.rb | 94 ++++----- docker-compose.prod.yml | 52 +++++ docker-compose.yml | 54 +++-- lib/capistrano/tasks/delayed_job.cap | 44 ----- lib/capistrano/tasks/logs.cap | 8 - 30 files changed, 349 insertions(+), 361 deletions(-) create mode 100644 .irbrc create mode 100644 Procfile.dev delete mode 100755 bin/delayed_job create mode 100755 bin/dev delete mode 100644 config/deploy/staging.rb delete mode 100644 config/initializers/delayed_job_config.rb create mode 100644 config/initializers/sidekiq.rb create mode 100644 config/sidekiq.yml create mode 100644 db/migrate/20240608163258_remove_delayed_job_table.rb create mode 100644 docker-compose.prod.yml delete mode 100644 lib/capistrano/tasks/delayed_job.cap delete mode 100644 lib/capistrano/tasks/logs.cap diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 8cecc79a..375401c0 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -7,24 +7,25 @@ on: jobs: test: runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - bundler-cache: true - - - run: RAILS_ENV=test bundle exec rake db:create - - run: RAILS_ENV=test bundle exec rake db:schema:load - - run: bundle exec rake - env: - COVERALLS_REPO_TOKEN : ${{ secrets.COVERALLS_REPO_TOKEN }} + - uses: actions/checkout@v4 + + - name: Build the stack + run: docker-compose up -d db-test + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + + - run: RAILS_ENV=test bundle exec rake db:create + - run: RAILS_ENV=test bundle exec rake db:schema:load + - run: bundle exec rake + lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Ruby uses: ruby/setup-ruby@v1 @@ -32,7 +33,5 @@ jobs: bundler-cache: true - name: Lint with rubocop - env: - RAILS_ENV: "test" run: | - bundle exec rubocop -c .rubocop.yml + bundle exec rubocop -c .rubocop.yml diff --git a/.irbrc b/.irbrc new file mode 100644 index 00000000..d31ae1d2 --- /dev/null +++ b/.irbrc @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require "rainbow/refinement" + +if defined? Rails + using Rainbow + + env_prompt = if Rails.env.development? + Rails.env[0..2].green + elsif Rails.env.production? + Rainbow(Rails.env.upcase).black.background(:red).bright.bold + else + Rails.env.magenta + end + + prompt = "[%n]#{env_prompt}(#{Rainbow('%m').yellow}):%i" + + IRB.conf[:PROMPT] ||= {} + IRB.conf[:PROMPT][:RAILS] = { + PROMPT_I: "#{prompt}> ", + PROMPT_S: "#{prompt}%l ", + PROMPT_C: "#{prompt}? ", + RETURN: "=> %s\n" + } + + IRB.conf[:PROMPT_MODE] = :RAILS +end + +IRB.conf[:USE_AUTOCOMPLETE] = ENV["IRB_USE_AUTOCOMPLETE"] == "true" diff --git a/.ruby-version b/.ruby-version index bea438e9..47725433 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -3.3.1 +3.3.2 diff --git a/.tool-versions b/.tool-versions index f76d2631..da68b39d 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,3 @@ -ruby 3.3.1 +ruby 3.3.2 yarn 1.22.19 nodejs 16.15.0 diff --git a/Gemfile b/Gemfile index 7c35d054..e8eb26e0 100644 --- a/Gemfile +++ b/Gemfile @@ -59,11 +59,6 @@ gem "cancancan", "~> 3.3" # Default avatar for users gem "identicon", "0.0.5" -# Run stuff in the background -gem "daemons", "~> 1.4" -gem "delayed_job", "~> 4.1" -gem "delayed_job_active_record" - # Needed for deployment somehow gem "bcrypt_pbkdf" gem "ed25519" @@ -81,10 +76,11 @@ gem "sentry-rails" gem "sentry-ruby" gem "stackprof" -# Production dependencies -group :production do - gem "mysql2", "~> 0.5.3" -end +# Run stuff in background +gem "sidekiq", "~> 7.0" + +# Use PostgreSQL as the database for Active Record +gem "pg" # Test dependencies group :test do @@ -94,9 +90,6 @@ end # Development dependencies group :development do - # Use sqlite3 as the database for Active Record - gem "sqlite3", "~> 1.4" - # Annotates Rails/ActiveRecord Models, routes, fixtures, and others based on the database schema. gem "annotate", "~> 3.1" @@ -104,10 +97,8 @@ group :development do gem "spring", "~> 3.0" # Deployment - gem "capistrano" - gem "capistrano-asdf" - gem "capistrano-passenger" - gem "capistrano-rails" + gem "capistrano", "~> 3.17" + gem "capistrano-docker", github: "TomNaessens/capistrano-docker" # Linting gem "rubocop", "~> 1.30" diff --git a/Gemfile.lock b/Gemfile.lock index bbc54ce2..4c5e2b87 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,60 +1,67 @@ +GIT + remote: https://github.com/TomNaessens/capistrano-docker.git + revision: 490311def501a5d32a24e52d6952d355a85b6cfe + specs: + capistrano-docker (0.4.0) + capistrano (>= 3.3) + GEM remote: https://rubygems.org/ specs: - actioncable (6.1.6) - actionpack (= 6.1.6) - activesupport (= 6.1.6) + actioncable (6.1.7.8) + actionpack (= 6.1.7.8) + activesupport (= 6.1.7.8) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (6.1.6) - actionpack (= 6.1.6) - activejob (= 6.1.6) - activerecord (= 6.1.6) - activestorage (= 6.1.6) - activesupport (= 6.1.6) + actionmailbox (6.1.7.8) + actionpack (= 6.1.7.8) + activejob (= 6.1.7.8) + activerecord (= 6.1.7.8) + activestorage (= 6.1.7.8) + activesupport (= 6.1.7.8) mail (>= 2.7.1) - actionmailer (6.1.6) - actionpack (= 6.1.6) - actionview (= 6.1.6) - activejob (= 6.1.6) - activesupport (= 6.1.6) + actionmailer (6.1.7.8) + actionpack (= 6.1.7.8) + actionview (= 6.1.7.8) + activejob (= 6.1.7.8) + activesupport (= 6.1.7.8) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (6.1.6) - actionview (= 6.1.6) - activesupport (= 6.1.6) + actionpack (6.1.7.8) + actionview (= 6.1.7.8) + activesupport (= 6.1.7.8) rack (~> 2.0, >= 2.0.9) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (6.1.6) - actionpack (= 6.1.6) - activerecord (= 6.1.6) - activestorage (= 6.1.6) - activesupport (= 6.1.6) + actiontext (6.1.7.8) + actionpack (= 6.1.7.8) + activerecord (= 6.1.7.8) + activestorage (= 6.1.7.8) + activesupport (= 6.1.7.8) nokogiri (>= 1.8.5) - actionview (6.1.6) - activesupport (= 6.1.6) + actionview (6.1.7.8) + activesupport (= 6.1.7.8) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.1, >= 1.2.0) - activejob (6.1.6) - activesupport (= 6.1.6) + activejob (6.1.7.8) + activesupport (= 6.1.7.8) globalid (>= 0.3.6) - activemodel (6.1.6) - activesupport (= 6.1.6) - activerecord (6.1.6) - activemodel (= 6.1.6) - activesupport (= 6.1.6) - activestorage (6.1.6) - actionpack (= 6.1.6) - activejob (= 6.1.6) - activerecord (= 6.1.6) - activesupport (= 6.1.6) + activemodel (6.1.7.8) + activesupport (= 6.1.7.8) + activerecord (6.1.7.8) + activemodel (= 6.1.7.8) + activesupport (= 6.1.7.8) + activestorage (6.1.7.8) + actionpack (= 6.1.7.8) + activejob (= 6.1.7.8) + activerecord (= 6.1.7.8) + activesupport (= 6.1.7.8) marcel (~> 1.0) mini_mime (>= 1.1.0) - activesupport (6.1.6) + activesupport (6.1.7.8) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -74,7 +81,7 @@ GEM bigdecimal (3.1.8) bootsnap (1.11.1) msgpack (~> 1.2) - builder (3.2.4) + builder (3.3.0) bulma-rails (0.9.3) sassc (~> 2.0) byebug (11.1.3) @@ -84,30 +91,15 @@ GEM i18n rake (>= 10.0.0) sshkit (>= 1.9.0) - capistrano-asdf (1.1.1) - capistrano (~> 3.0) - sshkit (~> 1.2) - capistrano-bundler (2.0.1) - capistrano (~> 3.1) - capistrano-passenger (0.2.1) - capistrano (~> 3.0) - capistrano-rails (1.6.2) - capistrano (~> 3.1) - capistrano-bundler (>= 1.1, < 3) chartkick (4.1.3) chunky_png (1.4.0) climate_control (0.2.0) - concurrent-ruby (1.2.3) + concurrent-ruby (1.3.2) + connection_pool (2.4.1) crack (0.4.5) rexml crass (1.0.6) - daemons (1.4.1) date (3.3.4) - delayed_job (4.1.10) - activesupport (>= 3.0, < 8.0) - delayed_job_active_record (4.1.7) - activerecord (>= 3.0, < 8.0) - delayed_job (>= 3.0, < 5) devise (4.8.1) bcrypt (~> 3.0) orm_adapter (~> 0.1) @@ -137,8 +129,8 @@ GEM sassc (>= 1.11) friendly_id (5.4.2) activerecord (>= 4.0.0) - globalid (1.0.0) - activesupport (>= 5.0) + globalid (1.2.1) + activesupport (>= 6.1) hashdiff (1.0.1) hashie (5.0.0) httparty (0.20.0) @@ -163,20 +155,22 @@ GEM loofah (2.22.0) crass (~> 1.0.2) nokogiri (>= 1.12.0) - mail (2.7.1) + mail (2.8.1) mini_mime (>= 0.1.1) - marcel (1.0.2) + net-imap + net-pop + net-smtp + marcel (1.0.4) method_source (1.1.0) mime-types (3.5.2) mime-types-data (~> 3.2015) mime-types-data (3.2024.0507) - mini_mime (1.1.2) + mini_mime (1.1.5) minitest (5.23.1) msgpack (1.5.1) multi_json (1.15.0) multi_xml (0.6.0) mutex_m (0.2.0) - mysql2 (0.5.4) net-imap (0.4.11) date net-protocol @@ -221,6 +215,7 @@ GEM parser (3.3.1.0) ast (~> 2.4.1) racc + pg (1.5.6) psych (5.1.2) stringio public_suffix (4.0.7) @@ -234,20 +229,20 @@ GEM rack rack-test (2.1.0) rack (>= 1.3) - rails (6.1.6) - actioncable (= 6.1.6) - actionmailbox (= 6.1.6) - actionmailer (= 6.1.6) - actionpack (= 6.1.6) - actiontext (= 6.1.6) - actionview (= 6.1.6) - activejob (= 6.1.6) - activemodel (= 6.1.6) - activerecord (= 6.1.6) - activestorage (= 6.1.6) - activesupport (= 6.1.6) + rails (6.1.7.8) + actioncable (= 6.1.7.8) + actionmailbox (= 6.1.7.8) + actionmailer (= 6.1.7.8) + actionpack (= 6.1.7.8) + actiontext (= 6.1.7.8) + actionview (= 6.1.7.8) + activejob (= 6.1.7.8) + activemodel (= 6.1.7.8) + activerecord (= 6.1.7.8) + activestorage (= 6.1.7.8) + activesupport (= 6.1.7.8) bundler (>= 1.15.0) - railties (= 6.1.6) + railties (= 6.1.7.8) sprockets-rails (>= 2.0.0) rails-controller-testing (1.0.5) actionpack (>= 5.0.1.rc1) @@ -260,9 +255,9 @@ GEM rails-html-sanitizer (1.6.0) loofah (~> 2.21) nokogiri (~> 1.14) - railties (6.1.6) - actionpack (= 6.1.6) - activesupport (= 6.1.6) + railties (6.1.7.8) + actionpack (= 6.1.7.8) + activesupport (= 6.1.7.8) method_source rake (>= 12.2) thor (~> 1.0) @@ -273,6 +268,8 @@ GEM ffi (~> 1.0) rdoc (6.7.0) psych (>= 4.0.0) + redis-client (0.22.2) + connection_pool regexp_parser (2.9.2) responders (3.0.1) actionpack (>= 5.0) @@ -334,17 +331,19 @@ GEM sentry-ruby (5.17.3) bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) + sidekiq (7.2.4) + concurrent-ruby (< 2) + connection_pool (>= 2.3.0) + rack (>= 2.2.4) + redis-client (>= 0.19.0) spring (3.1.1) - sprockets (4.0.3) + sprockets (4.2.1) concurrent-ruby (~> 1.0) - rack (> 1, < 3) - sprockets-rails (3.4.2) - actionpack (>= 5.2) - activesupport (>= 5.2) + rack (>= 2.2.4, < 4) + sprockets-rails (3.5.1) + actionpack (>= 6.1) + activesupport (>= 6.1) sprockets (>= 3.0.0) - sqlite3 (1.7.3-arm64-darwin) - sqlite3 (1.7.3-x86_64-darwin) - sqlite3 (1.7.3-x86_64-linux) sshkit (1.22.2) base64 mutex_m @@ -375,7 +374,7 @@ GEM rack-proxy (>= 0.6.1) railties (>= 5.2) semantic_range (>= 2.3.0) - websocket-driver (0.7.5) + websocket-driver (0.7.6) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) will_paginate (3.3.1) @@ -395,14 +394,9 @@ DEPENDENCIES bulma-rails (~> 0.9.1) byebug cancancan (~> 3.3) - capistrano - capistrano-asdf - capistrano-passenger - capistrano-rails + capistrano (~> 3.17) + capistrano-docker! chartkick (~> 4.0) - daemons (~> 1.4) - delayed_job (~> 4.1) - delayed_job_active_record devise (~> 4.8) dotenv-rails ed25519 @@ -415,12 +409,12 @@ DEPENDENCIES identicon (= 0.0.5) kt-paperclip (~> 7.0) listen (~> 3.7) - mysql2 (~> 0.5.3) net-imap net-pop net-smtp omniauth-rails_csrf_protection omniauth-zeuswpi + pg puma (~> 5.5.0) rails (~> 6.1) rails-controller-testing (~> 1.0) @@ -432,8 +426,8 @@ DEPENDENCIES sdoc (~> 2.2) sentry-rails sentry-ruby + sidekiq (~> 7.0) spring (~> 3.0) - sqlite3 (~> 1.4) stackprof tqdm (~> 0.3.0) tzinfo-data diff --git a/Procfile.dev b/Procfile.dev new file mode 100644 index 00000000..ebf1cbff --- /dev/null +++ b/Procfile.dev @@ -0,0 +1,3 @@ +web: PORT=3000 rails s +sidekiq: bundle exec sidekiq +db: docker-compose up diff --git a/README.md b/README.md index e685f5b1..ff757e76 100644 --- a/README.md +++ b/README.md @@ -7,47 +7,17 @@ To provide a consistent experience on every system, docker and docker-compose is used during development and production. -### Using Docker and Make *(recommended)* +1. Install the prerequisites: ruby `$(cat .ruby-version)`, preferably using [asdf](https://asdf-vm.com/), and some system libraries depending on your OS (e.g. imagemagick) +2. Install the ruby dependencies: `bin/bundle` +3. Start up the database, sidekiq and rails server by running `bin/dev` +4. Set up some database data using `rails db:setup` +5. Browse to http://localhost:3000 -#### Linux/Unix +In case you want to start the webserver in your IDE, just run `docker-compose up -d` and start Sidekiq manually (`bundle exec sidekiq`) -1. Install [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) -2. Start the development server - ```sh - make dev - ``` +## Deploying - > This will start a development server on http://localhost:3000 -3. Seed the database. - ```sh - make dev-seed - ``` - - > The development setup uses an SQLite 3 database, which can be found under `/db/development.sqlite3` - -#### Windows - -1. Install [WSL (Windows Subsystem for Linux)](https://docs.microsoft.com/en-us/windows/wsl/install-win10) -2. Follow the instructions for Linux/Unix above - -### Directly on your system - -1. Install [asdf](http://asdf-vm.com/guide/getting-started.html#getting-started) -2. Install dependencies: `asdf install` -3. Install gems: `bundle install` -4. Migrate the db using `bundle exec rails db:migrate` -5. Seed the db using `bundle exec rails db:seed` -6. Start Tap by running `bundle exec rails s` - -## Production - -You can generate a production docker image using: - -```sh -make build -``` - -> The image will be tagged under `tap:latest` +_Locally_, run `bundle exec cap production deploy` ## FAQ @@ -73,9 +43,12 @@ make build
There are no transactions going from Tap to Tab - The delay job may not be running. You can start it using: + Sidekiq might not be running. Check the dashboard on https://tap.zeus.gent/sidekiq. + + You can start by redeploying the application, or by turning the deployment on the server off & on (inside the `/home/tap/production/current` directory): - ```sh - sudo -u tap RAILS_ENV=production /home/tap/production/current/bin/delayed_job start + ```bash + podman-compose -f docker-compose.prod.yml -p tap down + podman-compose -f docker-compose.prod.yml -p tap up -d ```
diff --git a/app/controllers/concerns/statistics.rb b/app/controllers/concerns/statistics.rb index 3899dd0d..7536ec81 100644 --- a/app/controllers/concerns/statistics.rb +++ b/app/controllers/concerns/statistics.rb @@ -25,7 +25,7 @@ def products_group_by_category def products_group_by_id products .select("products.*", "sum(order_items.count) as count") - .group(:product_id) + .group("products.id") .order("count") .reverse_order end diff --git a/app/controllers/orders_controller.rb b/app/controllers/orders_controller.rb index 500fe383..14e4c7c8 100644 --- a/app/controllers/orders_controller.rb +++ b/app/controllers/orders_controller.rb @@ -152,7 +152,7 @@ def create # Create an order # POST(method: DELETE) /users/{username}/orders/{id} def destroy - @order.destroy + @order.destroy! respond_to do |format| format.json { head :no_content } format.html do diff --git a/app/jobs/tab_api_job.rb b/app/jobs/tab_api_job.rb index bdfca41f..5de519ea 100644 --- a/app/jobs/tab_api_job.rb +++ b/app/jobs/tab_api_job.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -TabApiJob = Struct.new(:order_id) do - def perform(*_args) +class TabApiJob < ApplicationJob + def perform(order_id) order = Order.find_by(id: order_id) return unless order && !order.transaction_id @@ -22,13 +22,11 @@ def perform(*_args) order.update(transaction_id: JSON.parse(result.body)["id"].to_i) end + private + def headers { "Authorization" => "Token token=#{Rails.application.secrets.tab_api_key}" } end - - def error(_job, exception) - Airbrake.notify(exception) - end end diff --git a/app/models/order.rb b/app/models/order.rb index daac94fe..c544c8ed 100644 --- a/app/models/order.rb +++ b/app/models/order.rb @@ -61,11 +61,7 @@ def calculate_price def create_api_job return if Rails.env.test? - priority = 0 - run_at = Rails.application.config.call_api_after.from_now - job = TabApiJob.new(id) - - Delayed::Job.enqueue job, priority: priority, run_at: run_at + TabApiJob.set(wait: Rails.application.config.call_api_after).perform_later(id) end def product_presence diff --git a/bin/delayed_job b/bin/delayed_job deleted file mode 100755 index edf19598..00000000 --- a/bin/delayed_job +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env ruby - -require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment')) -require 'delayed/command' -Delayed::Command.new(ARGV).daemonize diff --git a/bin/dev b/bin/dev new file mode 100755 index 00000000..72ff9603 --- /dev/null +++ b/bin/dev @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +docker-compose up -d + +if ! command -v foreman &> /dev/null +then + echo "Installing foreman…" + gem install foreman +fi + +foreman start -f Procfile.dev "$@" diff --git a/config/application.rb b/config/application.rb index 924e3f67..7f140a02 100644 --- a/config/application.rb +++ b/config/application.rb @@ -15,8 +15,8 @@ class Application < Rails::Application # Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC. config.time_zone = 'Brussels' - # Setup delayed jobs - config.active_job.queue_adapter = :delayed_job + # Setup Sidekiq + config.active_job.queue_adapter = :sidekiq config.call_api_after = 5.minutes config.frecency_num_orders = 25 diff --git a/config/database.yml b/config/database.yml index 1c1a37ca..d2a0dc7a 100644 --- a/config/database.yml +++ b/config/database.yml @@ -1,25 +1,20 @@ -# SQLite version 3.x -# gem install sqlite3 -# -# Ensure the SQLite 3 gem is defined in your Gemfile -# gem 'sqlite3' -# default: &default - adapter: sqlite3 - pool: 5 - timeout: 5000 + adapter: postgresql + pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> + host: 127.0.0.1 + username: postgres + password: password + encoding: utf8 + port: 5432 development: <<: *default - database: db/development.sqlite3 + database: tap -# Warning: The database defined as "test" will be erased and -# re-generated from your development database when you run "rake". -# Do not set this db to the same as development or production. test: <<: *default - database: db/test.sqlite3 + database: tap-test + port: 5433 production: <<: *default - database: db/production.sqlite3 diff --git a/config/deploy.rb b/config/deploy.rb index 4fbe44fb..83c20dcd 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -26,7 +26,7 @@ # set :pty, true # Default value for :linked_files is [] -set :linked_files, %w{config/database.yml config/secrets.yml .env} +set :linked_files, [] # Default value for linked_dirs is [] set :linked_dirs, %w{log tmp/pids tmp/cache tmp/sockets vendor/bundle public/system} @@ -37,16 +37,11 @@ # Default value for keep_releases is 5 # set :keep_releases, 5 -namespace :passenger do - desc "Restart Application" - task :restart do - on roles(:app) do - with rails_env: fetch(:rails_env) do - execute "touch #{current_path}/tmp/restart.txt" - end - end - invoke 'delayed_job:restart' - end -end - -after :deploy, "passenger:restart" +# capistrano-docker specific +set :docker_command, "podman" +set :docker_copy_data, %w{.env config/database.yml} + +set :docker_compose, true +set :docker_compose_path, "docker-compose.prod.yml" +set :docker_compose_command, "podman-compose" +set :docker_compose_project_name, "tap" diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb deleted file mode 100644 index 86752d05..00000000 --- a/config/deploy/staging.rb +++ /dev/null @@ -1,7 +0,0 @@ -server 'king.ugent.be', user: 'tap', roles: %w{web app db}, ssh_options: { - forward_agent: true, - auth_methods: ['publickey'], - port: 2222 -} - -set :rails_env, 'production' diff --git a/config/environments/development.rb b/config/environments/development.rb index 88611547..51f38058 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -80,5 +80,4 @@ config.api_url = "http://localhost:3001/api/v1/" config.tab_user_url = "http://localhost:3001/" - Delayed::Worker.logger = Rails.logger end diff --git a/config/initializers/delayed_job_config.rb b/config/initializers/delayed_job_config.rb deleted file mode 100644 index 7323b719..00000000 --- a/config/initializers/delayed_job_config.rb +++ /dev/null @@ -1,2 +0,0 @@ -Delayed::Worker.destroy_failed_jobs = false -Delayed::Worker.logger = Logger.new(File.join(Rails.root, 'log', 'delayed_job.log')) diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb new file mode 100644 index 00000000..c28d21a2 --- /dev/null +++ b/config/initializers/sidekiq.rb @@ -0,0 +1,3 @@ +Sidekiq.configure_server do |config| + config.redis = { url: ENV["REDIS_URL"] || "redis://localhost:6379" } +end diff --git a/config/routes.rb b/config/routes.rb index 290483bc..88645478 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -3,6 +3,12 @@ # This method is responsible to generate all needed routes for devise, based on what modules you have defined in your model. devise_for :users, controllers: { omniauth_callbacks: "callbacks" } + # Sidekiq UI + require 'sidekiq/web' + authenticate :user, -> (user) { user.admin? } do + mount Sidekiq::Web => '/sidekiq' + end + # Authentication # Using "devise" devise_scope :user do diff --git a/config/secrets.yml b/config/secrets.yml index 388228df..6636e459 100644 --- a/config/secrets.yml +++ b/config/secrets.yml @@ -14,20 +14,17 @@ development: secret_key_base: 5d40610321e19e4f71ee2ba8af4f426fe15096c405da3800c6b33bed6779f2d11f55a0edc455974b19a01fd71f6cd508dba980305dbc55ff82521a2d12f891d8 omniauth_client_id: tomtest omniauth_client_secret: blargh - access_token: "token" tab_api_key: "o6UIYc0r0cvsy0Ohaohqsw==" koelkast_token: "token" test: secret_key_base: 961437e28e7d6055ffaad9cf1f8d614354f57f10cb2d7601c9d6ede72a03b9c9535ad9e63507e3eb31252c4895970a63117493408f2e9a46c7a0c4a5a7836b81 - access_token: "token" # Do not keep production secrets in the repository, # instead read values from the environment. production: secret_key_base: <%= ENV["SECRET_KEY_BASE"] %> - omniauth_client_id: "" - omniauth_client_secret: "" - access_token: "" - tab_api_key: "" - koelkast_token: "" + omniauth_client_id: <%= ENV["OMNIAUTH_CLIENT_ID"] %> + omniauth_client_secret: <%= ENV["OMNIAUTH_CLIENT_SECRET"] %> + tab_api_key: <%= ENV["TAB_API_KEY"] %> + koelkast_token: <%= ENV["KOEKAST_TOKEN"] %> diff --git a/config/sidekiq.yml b/config/sidekiq.yml new file mode 100644 index 00000000..b68d2e2d --- /dev/null +++ b/config/sidekiq.yml @@ -0,0 +1,5 @@ +--- +:concurrency: 1 +:queues: + - default + - mailers diff --git a/db/migrate/20240608163258_remove_delayed_job_table.rb b/db/migrate/20240608163258_remove_delayed_job_table.rb new file mode 100644 index 00000000..f8b46e96 --- /dev/null +++ b/db/migrate/20240608163258_remove_delayed_job_table.rb @@ -0,0 +1,5 @@ +class RemoveDelayedJobTable < ActiveRecord::Migration[6.1] + def up + drop_table :delayed_jobs + end +end diff --git a/db/schema.rb b/db/schema.rb index 1af54ff9..b1551c38 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,60 +10,51 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2022_07_25_163014) do +ActiveRecord::Schema.define(version: 2024_06_08_163258) do - create_table "barcodes", force: :cascade do |t| - t.integer "product_id", null: false - t.string "code", default: "", null: false - t.datetime "created_at" - t.datetime "updated_at" - t.index ["code"], name: "index_barcodes_on_code" - end + # These are extensions that must be enabled in order to support this database + enable_extension "plpgsql" - create_table "delayed_jobs", force: :cascade do |t| - t.integer "priority", default: 0, null: false - t.integer "attempts", default: 0, null: false - t.text "handler", null: false - t.text "last_error" - t.datetime "run_at" - t.datetime "locked_at" - t.datetime "failed_at" - t.string "locked_by" - t.string "queue" + create_table "barcodes", force: :cascade do |t| + t.bigint "product_id", null: false + t.string "code", limit: 255, default: "", null: false t.datetime "created_at" t.datetime "updated_at" - t.index ["priority", "run_at"], name: "delayed_jobs_priority" + t.index ["code"], name: "idx_16395_index_barcodes_on_code" + t.index ["product_id"], name: "idx_16395_fk_rails_f6f6672052" end create_table "order_items", force: :cascade do |t| - t.integer "order_id", null: false - t.integer "product_id", null: false - t.integer "count", default: 0 + t.bigint "order_id", null: false + t.bigint "product_id", null: false + t.bigint "count", default: 0 + t.index ["order_id"], name: "idx_16419_fk_rails_e3cb28f071" + t.index ["product_id"], name: "idx_16419_fk_rails_f1a29ddd47" end create_table "orders", force: :cascade do |t| - t.integer "user_id", null: false - t.integer "price_cents" + t.bigint "user_id", null: false + t.bigint "price_cents" t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.integer "transaction_id" - t.index ["created_at"], name: "index_orders_on_created_at" - t.index ["user_id", "created_at"], name: "index_orders_on_user_id_and_created_at" - t.index ["user_id"], name: "index_orders_on_user_id" + t.bigint "transaction_id" + t.index ["created_at"], name: "idx_16414_index_orders_on_created_at" + t.index ["user_id", "created_at"], name: "idx_16414_index_orders_on_user_id_and_created_at" + t.index ["user_id"], name: "idx_16414_index_orders_on_user_id" end create_table "products", force: :cascade do |t| - t.string "name", null: false - t.integer "price_cents", default: 0, null: false + t.string "name", limit: 255, null: false + t.bigint "price_cents", default: 0, null: false t.datetime "created_at" t.datetime "updated_at" - t.string "avatar_file_name" - t.string "avatar_content_type" - t.integer "avatar_file_size" + t.string "avatar_file_name", limit: 255 + t.string "avatar_content_type", limit: 255 + t.bigint "avatar_file_size" t.datetime "avatar_updated_at" - t.integer "category", default: 0 - t.integer "stock", default: 0, null: false - t.integer "calories" + t.bigint "category", default: 0 + t.bigint "stock", default: 0, null: false + t.bigint "calories" t.boolean "deleted", default: false end @@ -72,25 +63,26 @@ t.datetime "updated_at" t.datetime "remember_created_at" t.boolean "admin", default: false - t.integer "dagschotel_id" - t.string "avatar_file_name" - t.string "avatar_content_type" - t.integer "avatar_file_size" + t.bigint "dagschotel_id" + t.string "avatar_file_name", limit: 255 + t.string "avatar_content_type", limit: 255 + t.bigint "avatar_file_size" t.datetime "avatar_updated_at" - t.integer "orders_count", default: 0 + t.bigint "orders_count", default: 0 t.boolean "koelkast", default: false - t.string "name" + t.string "name", limit: 255 t.boolean "private", default: false - t.integer "frecency", default: 0, null: false + t.bigint "frecency", default: 0, null: false t.boolean "quickpay_hidden", default: false - t.string "userkey" - t.index ["koelkast"], name: "index_users_on_koelkast" - t.index ["orders_count"], name: "index_users_on_orders_count" + t.string "userkey", limit: 255 + t.index ["dagschotel_id"], name: "idx_16441_fk_rails_b21d65e995" + t.index ["koelkast"], name: "idx_16441_index_users_on_koelkast" + t.index ["orders_count"], name: "idx_16441_index_users_on_orders_count" end - add_foreign_key "barcodes", "products" - add_foreign_key "order_items", "orders" - add_foreign_key "order_items", "products" - add_foreign_key "orders", "users" - add_foreign_key "users", "products", column: "dagschotel_id" + add_foreign_key "barcodes", "products", on_update: :restrict, on_delete: :restrict + add_foreign_key "order_items", "orders", on_update: :restrict, on_delete: :restrict + add_foreign_key "order_items", "products", on_update: :restrict, on_delete: :restrict + add_foreign_key "orders", "users", on_update: :restrict, on_delete: :restrict + add_foreign_key "users", "products", column: "dagschotel_id", on_update: :restrict, on_delete: :restrict end diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 00000000..699c72ca --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,52 @@ +version: '3.3' + +services: + db: + image: postgres:16 + environment: + POSTGRES_DB: tap + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + volumes: + - tap_data:/var/lib/postgresql/data + ports: + - "5432:5432" + + web: + build: . + depends_on: + - db + ports: + - "3008:3000" + volumes: + - paperclip_storage:/app/public/system + environment: + DATABASE_URL: postgresql://postgres:password@db/tap + RAILS_LOG_TO_STDOUT: 1 + SECRET_AUTH_USERNAME: admin + SECRET_AUTH_PASSWORD: admin-password + RAILS_SERVE_STATIC_FILES: 1 + + sidekiq: + build: . + depends_on: + - db + - redis + volumes: + - paperclip_storage:/app/public/system + environment: + DATABASE_URL: postgresql://postgres:password@db/tap + REDIS_URL: redis://redis:6379 + entrypoint: bundle exec sidekiq + + redis: + image: redis:7-alpine + volumes: + - redis_data:/data + ports: + - "6379:6379" + +volumes: + tap_data: + redis_data: + paperclip_storage: diff --git a/docker-compose.yml b/docker-compose.yml index fefe7a73..f9d2427e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,25 +1,37 @@ # -# This compose file is meant for development and should not be used in production! -# -version: "3" +version: '3.3' + services: - # Development service - development: - build: - context: . - # Build the image using the host network - # This is to allow building the image on eduroam (since it clashes with the default docker range) - network: host - # Development build - target: development - ports: - # adams.ugent.be only support port 3000 for authentication in localhost - - "3000:80" + db: + image: postgres:16 + environment: + POSTGRES_DB: tap + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password volumes: - # Mount the source code for development - - "./:/tap/" + - tap_data:/var/lib/postgresql/data + ports: + - "5432:5432" + + db-test: + image: postgres:16 environment: - # Enable docker buildkit caching between layers - BUILDKIT_INLINE_CACHE: 1 - tmpfs: - - /tap/tmp + POSTGRES_DB: tap-test + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + volumes: + - tap_test_data:/var/lib/postgresql/data + ports: + - "5433:5432" + + redis: + image: redis:7-alpine + volumes: + - redis_data:/data + ports: + - "6379:6379" + +volumes: + tap_data: + tap_test_data: + redis_data: diff --git a/lib/capistrano/tasks/delayed_job.cap b/lib/capistrano/tasks/delayed_job.cap deleted file mode 100644 index 3a206b66..00000000 --- a/lib/capistrano/tasks/delayed_job.cap +++ /dev/null @@ -1,44 +0,0 @@ -namespace :delayed_job do - - def args - fetch(:delayed_job_args, "") - end - - def delayed_job_roles - fetch(:delayed_job_server_role, :app) - end - - desc 'Stop the delayed_job process' - task :stop do - on roles(delayed_job_roles) do - within release_path do - with rails_env: fetch(:rails_env) do - execute :ruby, 'bin/delayed_job', :stop - end - end - end - end - - desc 'Start the delayed_job process' - task :start do - on roles(delayed_job_roles) do - within release_path do - with rails_env: fetch(:rails_env) do - execute :ruby, 'bin/delayed_job', args, :start - end - end - end - end - - desc 'Restart the delayed_job process' - task :restart do - on roles(delayed_job_roles) do - within release_path do - with rails_env: fetch(:rails_env) do - execute :ruby, 'bin/delayed_job', args, :restart - end - end - end - end - -end diff --git a/lib/capistrano/tasks/logs.cap b/lib/capistrano/tasks/logs.cap deleted file mode 100644 index 4855a5d8..00000000 --- a/lib/capistrano/tasks/logs.cap +++ /dev/null @@ -1,8 +0,0 @@ -namespace :logs do - desc "tail rails logs" - task :tail do - on roles(:app) do - execute "tail -f #{shared_path}/log/#{fetch(:rails_env)}.log" - end - end -end From 5dda46de697865afbf019859983e5c89f7ea7445 Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sat, 8 Jun 2024 21:33:13 +0200 Subject: [PATCH 02/26] Correct Capfile --- Capfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Capfile b/Capfile index 5876b1fe..2aad4798 100644 --- a/Capfile +++ b/Capfile @@ -9,8 +9,9 @@ require "capistrano/deploy" require "capistrano/scm/git" install_plugin Capistrano::SCM::Git -require "capistrano/rails" -require "capistrano/asdf" +require "capistrano/docker" +require "capistrano/docker/compose/logs" +require "capistrano/docker/compose/migration" # Loads custom tasks from `lib/capistrano/tasks' if you have any defined. -Dir.glob("lib/capistrano/tasks/*.cap").each { |r| import r } +Dir.glob("lib/capistrano/tasks/*.rake").each { |r| import r } From dbadae2ae7c2a90ac350caa7f86b33c2196105e1 Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sat, 8 Jun 2024 21:34:51 +0200 Subject: [PATCH 03/26] Correct Dockerfile --- Dockerfile | 69 ++++++++---------------------------------------------- 1 file changed, 10 insertions(+), 59 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3bf26b53..cf9a8647 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,67 +1,18 @@ -################## -### Base image ### -################## -FROM ruby:3.0.2-alpine as base +FROM ruby:3.3.2 -# Install the required packages for building -# Delete APK cache at the end (for smaller production images) -RUN apk update && \ - apk add --virtual build-dependencies build-base && \ - apk add shared-mime-info mariadb-dev sqlite-dev nodejs tzdata imagemagick && \ - rm -rf /var/cache/apk/* +ENV RAILS_ENV=production -# Create a working directory -WORKDIR /tap +RUN apt update && apt install -y nodejs cron -# Copy the gemfile to the working directory -COPY Gemfile Gemfile -COPY Gemfile.lock Gemfile.lock +WORKDIR /app -# Install dependencies -# Use BuildKit cache for caching dependencies -RUN --mount=type=cache,target=vendor/cache bundle install +COPY ./Gemfile ./Gemfile.lock /app/ -# Copy all of the .gem files needed to run the application into the vendor/cache directory. -# In the future, when running [bundle install(1)][bundle-install], use the gems in the cache in preference to the ones on rubygems.org -RUN bundle cache +RUN gem install bundler +RUN bundle install -######################## -### Production image ### -######################## -FROM base as production +COPY . /app -# Copy the sourcecode -COPY . . +RUN bundle exec rails assets:precompile -# Run rails in production mode -ENV RAILS_ENV production - -# Expose port 80 -# This is the main port for the application -EXPOSE 80 - -# Pre-compile assets -RUN rake assets:precompile - -# Docker Entrypoint -# Will be started when the container is started -ENTRYPOINT sh docker-start.sh - -######################### -### Development image ### -######################### -FROM base as development - -# Copy the sourcecode -COPY . . - -# Run rails in production mode -ENV RAILS_ENV development - -# Expose port 80 -# This is the main port for the application -EXPOSE 80 - -# Docker Entrypoint -# Will be started when the container is started -CMD sh docker-start.sh +CMD bundle exec rails s -b 0.0.0.0 From 6925bce58c486f8bd634590e95a947eb56ef28a3 Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sat, 8 Jun 2024 22:46:33 +0200 Subject: [PATCH 04/26] Don't build development and test gems --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index cf9a8647..d303f0ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,8 @@ WORKDIR /app COPY ./Gemfile ./Gemfile.lock /app/ RUN gem install bundler + +RUN bundle config set without 'development test' RUN bundle install COPY . /app From ea6cb08ce23fbe1bed4b236a2e38a2398750cc07 Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sat, 8 Jun 2024 23:51:12 +0200 Subject: [PATCH 05/26] Get this thing to build on 3.3.2-slim --- Dockerfile | 11 ++++++++--- Gemfile | 3 --- Gemfile.lock | 4 +--- yarn.lock | 29 +++++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index d303f0ed..8aece358 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,23 @@ -FROM ruby:3.3.2 +FROM ruby:3.3.2-slim ENV RAILS_ENV=production -RUN apt update && apt install -y nodejs cron +RUN apt update && apt install -y git libpq-dev build-essential gpg curl WORKDIR /app COPY ./Gemfile ./Gemfile.lock /app/ RUN gem install bundler - RUN bundle config set without 'development test' RUN bundle install +RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - +RUN apt install -y nodejs +RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - +RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list +RUN apt update && apt install -y yarn + COPY . /app RUN bundle exec rails assets:precompile diff --git a/Gemfile b/Gemfile index e8eb26e0..d708c7a3 100644 --- a/Gemfile +++ b/Gemfile @@ -63,9 +63,6 @@ gem "identicon", "0.0.5" gem "bcrypt_pbkdf" gem "ed25519" -# Ruby debugger -gem "byebug" - gem "httparty", "~> 0.20.0" # windows support (linux timezone directory is used in the project) diff --git a/Gemfile.lock b/Gemfile.lock index 4c5e2b87..b6dcb12d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -84,7 +84,6 @@ GEM builder (3.3.0) bulma-rails (0.9.3) sassc (~> 2.0) - byebug (11.1.3) cancancan (3.3.0) capistrano (3.18.1) airbrussh (>= 1.0.0) @@ -392,7 +391,6 @@ DEPENDENCIES bcrypt_pbkdf bootsnap (~> 1.9) bulma-rails (~> 0.9.1) - byebug cancancan (~> 3.3) capistrano (~> 3.17) capistrano-docker! @@ -438,4 +436,4 @@ DEPENDENCIES will_paginate-bulma (~> 1.0) BUNDLED WITH - 2.5.10 + 2.5.11 diff --git a/yarn.lock b/yarn.lock index 4b44e415..099982df 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7283,6 +7283,35 @@ webpack@^4.46.0: watchpack "^1.7.4" webpack-sources "^1.4.1" +webpack@^4.47.0: + version "4.47.0" + resolved "https://registry.yarnpkg.com/webpack/-/webpack-4.47.0.tgz#8b8a02152d7076aeb03b61b47dad2eeed9810ebc" + integrity sha512-td7fYwgLSrky3fI1EuU5cneU4+pbH6GgOfuKNS1tNPcfdGinGELAqsb/BP4nnvZyKSG2i/xFGU7+n2PvZA8HJQ== + dependencies: + "@webassemblyjs/ast" "1.9.0" + "@webassemblyjs/helper-module-context" "1.9.0" + "@webassemblyjs/wasm-edit" "1.9.0" + "@webassemblyjs/wasm-parser" "1.9.0" + acorn "^6.4.1" + ajv "^6.10.2" + ajv-keywords "^3.4.1" + chrome-trace-event "^1.0.2" + enhanced-resolve "^4.5.0" + eslint-scope "^4.0.3" + json-parse-better-errors "^1.0.2" + loader-runner "^2.4.0" + loader-utils "^1.2.3" + memory-fs "^0.4.1" + micromatch "^3.1.10" + mkdirp "^0.5.3" + neo-async "^2.6.1" + node-libs-browser "^2.2.1" + schema-utils "^1.0.0" + tapable "^1.1.3" + terser-webpack-plugin "^1.4.3" + watchpack "^1.7.4" + webpack-sources "^1.4.1" + websocket-driver@>=0.5.1, websocket-driver@^0.7.4: version "0.7.4" resolved "https://registry.yarnpkg.com/websocket-driver/-/websocket-driver-0.7.4.tgz#89ad5295bbf64b480abcba31e4953aca706f5760" From 38205841bcc2915745a49f99401a753ec1aa9c08 Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sun, 9 Jun 2024 16:18:48 +0200 Subject: [PATCH 06/26] Add build github action --- .github/workflows/build.yml | 26 ++++++++++++++++++++++++++ Dockerfile | 3 +++ 2 files changed, 29 insertions(+) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..5ed4b802 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,26 @@ +name: Build + +on: + push: + branches: + - 'main' + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - + name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - + name: Build and push + uses: docker/build-push-action@v5 + with: + push: true diff --git a/Dockerfile b/Dockerfile index 8aece358..7019839c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,9 @@ RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list RUN apt update && apt install -y yarn +COPY ./package.json ./yarn.lock /app/ +RUN yarn install + COPY . /app RUN bundle exec rails assets:precompile From 578b92ce85318e96fefb2dd0766455f26e061be2 Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sun, 9 Jun 2024 16:20:51 +0200 Subject: [PATCH 07/26] Master, not main --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5ed4b802..89da686d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,7 +3,7 @@ name: Build on: push: branches: - - 'main' + - 'master' jobs: docker: From 965dd5c26843c53f287aaeff55e88caced550b22 Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sun, 9 Jun 2024 16:35:49 +0200 Subject: [PATCH 08/26] Add labels and tags --- .github/workflows/build.yml | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 89da686d..dd9a1c45 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,18 +9,33 @@ jobs: docker: runs-on: ubuntu-latest steps: - - - name: Set up Docker Buildx + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ZeusWPI/Tap + ghcr.io/ZeusWPI/Tap + tags: | + type=sha + flavor: | + latest=auto + + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub Container Registry + + - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io - username: ${{ github.actor }} + username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push + + - name: Build and push uses: docker/build-push-action@v5 with: + context: . + file: ./Dockerfile push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} From 1384d3a55047cae4dbbd64e46b772522d764b035 Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sun, 9 Jun 2024 16:38:35 +0200 Subject: [PATCH 09/26] Use git context --- .github/workflows/build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dd9a1c45..0437a91f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,8 +34,6 @@ jobs: - name: Build and push uses: docker/build-push-action@v5 with: - context: . - file: ./Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From c63330ec1506a7f14a037b1ecd78a71cf2c1bb32 Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sun, 9 Jun 2024 18:02:14 +0200 Subject: [PATCH 10/26] Add SECRET_KEY_BASE secret --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0437a91f..66db227b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,6 +33,8 @@ jobs: - name: Build and push uses: docker/build-push-action@v5 + env: + SECRET_KEY_BASE: ${{ secrets.SECRET_KEY_BASE }} with: push: true tags: ${{ steps.meta.outputs.tags }} From 34ef51599c4dd0b9b1c1708bc84a1bd678cb240f Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sun, 9 Jun 2024 18:13:45 +0200 Subject: [PATCH 11/26] Explicitly call it SECRET_KEY_BASE_DUMMY --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 66db227b..0c99808d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,7 +34,7 @@ jobs: - name: Build and push uses: docker/build-push-action@v5 env: - SECRET_KEY_BASE: ${{ secrets.SECRET_KEY_BASE }} + SECRET_KEY_BASE: ${{ secrets.SECRET_KEY_BASE_DUMMY }} with: push: true tags: ${{ steps.meta.outputs.tags }} From 7f5170f2a5f9bdc795e9600e5214bd4538a04161 Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sun, 9 Jun 2024 18:21:08 +0200 Subject: [PATCH 12/26] Use dummy secret key base directly --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7019839c..bea7255c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,6 @@ RUN yarn install COPY . /app -RUN bundle exec rails assets:precompile +RUN SECRET_KEY_BASE="dummy_secret_key_base" bundle exec rails assets:precompile CMD bundle exec rails s -b 0.0.0.0 From 1c112a8adc684723c4c8c6a4c10669d08d939b13 Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sun, 9 Jun 2024 18:36:10 +0200 Subject: [PATCH 13/26] Set registry to ghcr.io --- .github/workflows/build.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0c99808d..6f7bde7c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,6 +5,9 @@ on: branches: - 'master' +env: + REGISTRY: ghcr.io + jobs: docker: runs-on: ubuntu-latest @@ -27,15 +30,14 @@ jobs: - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: - registry: ghcr.io + registry: ${{ env.REGISTRY }} username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push uses: docker/build-push-action@v5 - env: - SECRET_KEY_BASE: ${{ secrets.SECRET_KEY_BASE_DUMMY }} with: + registry: ghcr.io push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 80dd528ea6913b1d99cf8a98fe476462aa2083ad Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sun, 9 Jun 2024 18:43:37 +0200 Subject: [PATCH 14/26] What if we remove the default label, will it then only push to GHCR? --- .github/workflows/build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6f7bde7c..2be4208f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,6 @@ jobs: uses: docker/metadata-action@v5 with: images: | - ZeusWPI/Tap ghcr.io/ZeusWPI/Tap tags: | type=sha @@ -37,7 +36,7 @@ jobs: - name: Build and push uses: docker/build-push-action@v5 with: - registry: ghcr.io + registry: ${{ env.REGISTRY }} push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 39862caf23cd1be70edc630555495816ec27be75 Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sun, 9 Jun 2024 18:50:39 +0200 Subject: [PATCH 15/26] Remove registry, add cache to build workflow --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2be4208f..c9a09701 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,6 +17,7 @@ jobs: uses: docker/metadata-action@v5 with: images: | + ZeusWPI/Tap ghcr.io/ZeusWPI/Tap tags: | type=sha @@ -36,7 +37,8 @@ jobs: - name: Build and push uses: docker/build-push-action@v5 with: - registry: ${{ env.REGISTRY }} push: true + cache-from: type=gha + cache-to: type=gha,mode=max tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 9572634fbf74ec3dbd24a6b99404d50e0d572672 Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sun, 9 Jun 2024 18:54:53 +0200 Subject: [PATCH 16/26] Update docker-compose to pull from ghcr.io --- .github/workflows/build.yml | 1 - docker-compose.prod.yml | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c9a09701..e36de5a0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,6 @@ jobs: uses: docker/metadata-action@v5 with: images: | - ZeusWPI/Tap ghcr.io/ZeusWPI/Tap tags: | type=sha diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 699c72ca..20113b17 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -13,9 +13,7 @@ services: - "5432:5432" web: - build: . - depends_on: - - db + image: "ghcr.io/ZeusWPI/Tap:latest" ports: - "3008:3000" volumes: @@ -28,7 +26,7 @@ services: RAILS_SERVE_STATIC_FILES: 1 sidekiq: - build: . + image: "ghcr.io/ZeusWPI/Tap:latest" depends_on: - db - redis From 381e6b39aebe2b1cfb92b5034fb9f756b94d5a6a Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sun, 9 Jun 2024 19:09:30 +0200 Subject: [PATCH 17/26] Error: repository name must be lowercase --- docker-compose.prod.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 20113b17..9c2d035d 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -13,7 +13,7 @@ services: - "5432:5432" web: - image: "ghcr.io/ZeusWPI/Tap:latest" + image: "ghcr.io/zeuswpi/tap:latest" ports: - "3008:3000" volumes: @@ -26,7 +26,7 @@ services: RAILS_SERVE_STATIC_FILES: 1 sidekiq: - image: "ghcr.io/ZeusWPI/Tap:latest" + image: "ghcr.io/zeuswpi/tap:latest" depends_on: - db - redis From 27a0bf92524811065c72e778ace7b8ef1bb94f3f Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sun, 9 Jun 2024 19:15:35 +0200 Subject: [PATCH 18/26] Tag latest build as latest --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e36de5a0..ce06f60a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,6 +19,7 @@ jobs: images: | ghcr.io/ZeusWPI/Tap tags: | + type=raw,value=latest,enable={{is_default_branch}} type=sha flavor: | latest=auto From af6b26950ce9c738a2171200357abf948d6c8899 Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sun, 9 Jun 2024 19:20:30 +0200 Subject: [PATCH 19/26] Do we need to publicly expose the db port? --- docker-compose.prod.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 9c2d035d..c48b10ee 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -9,8 +9,6 @@ services: POSTGRES_PASSWORD: password volumes: - tap_data:/var/lib/postgresql/data - ports: - - "5432:5432" web: image: "ghcr.io/zeuswpi/tap:latest" From 35c5bc2da4fa943fb5a73e07a90aa84885473d6d Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sun, 9 Jun 2024 19:36:29 +0200 Subject: [PATCH 20/26] Pass .env file to docker-compose services --- docker-compose.prod.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index c48b10ee..7b164549 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -22,6 +22,8 @@ services: SECRET_AUTH_USERNAME: admin SECRET_AUTH_PASSWORD: admin-password RAILS_SERVE_STATIC_FILES: 1 + env_file: + - .env sidekiq: image: "ghcr.io/zeuswpi/tap:latest" @@ -34,6 +36,8 @@ services: DATABASE_URL: postgresql://postgres:password@db/tap REDIS_URL: redis://redis:6379 entrypoint: bundle exec sidekiq + env_file: + - .env redis: image: redis:7-alpine From bdf84ae532b5695c41649a73e7f53bcea08355cd Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sun, 9 Jun 2024 21:04:37 +0200 Subject: [PATCH 21/26] Add rainbow for colored output on production --- Gemfile | 3 +++ Gemfile.lock | 1 + 2 files changed, 4 insertions(+) diff --git a/Gemfile b/Gemfile index d708c7a3..26c7814b 100644 --- a/Gemfile +++ b/Gemfile @@ -79,6 +79,9 @@ gem "sidekiq", "~> 7.0" # Use PostgreSQL as the database for Active Record gem "pg" +# Rainbow for colored output in irb, even in production +gem "rainbow" + # Test dependencies group :test do gem "rspec-rails" diff --git a/Gemfile.lock b/Gemfile.lock index b6dcb12d..2768743c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -416,6 +416,7 @@ DEPENDENCIES puma (~> 5.5.0) rails (~> 6.1) rails-controller-testing (~> 1.0) + rainbow rspec-rails rubocop (~> 1.30) rubocop-rails (~> 2.14) From 467bd95d1b4e626a4765f06970fd6ce6b4c92253 Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sun, 9 Jun 2024 21:26:35 +0200 Subject: [PATCH 22/26] Set pull_policy to always --- docker-compose.prod.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 7b164549..5ad8f18c 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -12,6 +12,7 @@ services: web: image: "ghcr.io/zeuswpi/tap:latest" + pull_policy: always ports: - "3008:3000" volumes: @@ -27,6 +28,7 @@ services: sidekiq: image: "ghcr.io/zeuswpi/tap:latest" + pull_policy: always depends_on: - db - redis From dc470abcef2c2adf2750bb4cc75b0d52ce6d681f Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sun, 9 Jun 2024 21:35:40 +0200 Subject: [PATCH 23/26] Get rid of caching if podman can't handle it --- .github/workflows/build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ce06f60a..10288845 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,7 +38,5 @@ jobs: uses: docker/build-push-action@v5 with: push: true - cache-from: type=gha - cache-to: type=gha,mode=max tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 22ab3689320898614e563e1d732cc9b322213dc7 Mon Sep 17 00:00:00 2001 From: Tom Naessens Date: Sun, 9 Jun 2024 21:58:43 +0200 Subject: [PATCH 24/26] Stop exposing redis port --- app/models/user.rb | 2 +- docker-compose.prod.yml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/app/models/user.rb b/app/models/user.rb index c79f9a21..7fc1d06b 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -113,7 +113,7 @@ def generate_key def generate_key! set_key - save + save! end private diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 5ad8f18c..ca3f9283 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -45,8 +45,6 @@ services: image: redis:7-alpine volumes: - redis_data:/data - ports: - - "6379:6379" volumes: tap_data: From ff1b6d463c0feb138b9459ef7cdc79f5dfc9ffa2 Mon Sep 17 00:00:00 2001 From: Xander Bil Date: Sun, 9 Jun 2024 22:33:23 +0200 Subject: [PATCH 25/26] use alpine --- docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index ca3f9283..6d7158e3 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -2,7 +2,7 @@ version: '3.3' services: db: - image: postgres:16 + image: postgres:16-alpine environment: POSTGRES_DB: tap POSTGRES_USER: postgres From 95b5c94e9480227db36153dbc64e0772db8cbe48 Mon Sep 17 00:00:00 2001 From: Xander Bil Date: Sun, 9 Jun 2024 22:53:24 +0200 Subject: [PATCH 26/26] test building on this branch --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 10288845..a110e57a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,6 +4,7 @@ on: push: branches: - 'master' + - 'tom-containerize' env: REGISTRY: ghcr.io