From 78ffdd94d3e98b7eb88df072909a7228e6875c1c Mon Sep 17 00:00:00 2001 From: bibendi Date: Mon, 10 Nov 2014 09:44:33 +0600 Subject: [PATCH] add scheduler, retry, multy-job-forks --- CHANGELOG | 0 Gemfile | 1 + Makefile | 15 ++++++ README.md | 47 +++++++++++++++++++ Rakefile | 8 +++- VERSION | 1 + lib/resque/integration.rb | 25 +++++++++- lib/resque/integration/configuration.rb | 37 +++++++++++++-- lib/resque/integration/engine.rb | 16 ++++++- lib/resque/integration/god.erb | 46 +++++++++++++++++- lib/resque/integration/tasks/hooks.rake | 10 ++-- lib/resque/integration/tasks/supervisor.rake | 14 ------ lib/resque/integration/unique.rb | 16 ++++++- resque-integration.gemspec | 10 ++-- spec/resque/integration/configuration_spec.rb | 43 ++++++++++++++++- 15 files changed, 257 insertions(+), 32 deletions(-) create mode 100644 CHANGELOG create mode 100644 Makefile create mode 100644 VERSION delete mode 100644 lib/resque/integration/tasks/supervisor.rake diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000..e69de29 diff --git a/Gemfile b/Gemfile index f6ac03c..877aa32 100644 --- a/Gemfile +++ b/Gemfile @@ -1,3 +1,4 @@ +source 'http://apress:montalcino@gems.railsc.ru' source 'https://rubygems.org' # Specify your gem's dependencies in resque-integration.gemspec diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..9eb58aa --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +RAILS_ENV = test +BUNDLE = RAILS_ENV=${RAILS_ENV} bundle +BUNDLE_OPTIONS = -j 2 +RSPEC = rspec + +all: test + +test: bundler/install + ${BUNDLE} exec ${RSPEC} spec 2>&1 + +bundler/install: + if ! gem list bundler -i > /dev/null; then \ + gem install bundler; \ + fi + ${BUNDLE} install ${BUNDLE_OPTIONS} diff --git a/README.md b/README.md index fd826c4..ff7ed02 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,14 @@ # Resque::Integration + + Интеграция Resque в Rails-приложения с поддержкой следующих плагинов: * [resque-progress](https://github.com/idris/resque-progress) * [resque-lock](https://github.com/defunkt/resque-lock) * [resque-multi-job-forks](https://github.com/stulentsev/resque-multi-job-forks) * [resque-failed-job-mailer](https://github.com/anandagrawal84/resque_failed_job_mailer) +* [resque-scheduler](https://github.com/resque/resque-scheduler) +* [resque-retry](https://github.com/lantins/resque-retry) Этот гем существует затем, чтобы избежать повторения чужих ошибок и сократить время, необходимое для включения resque в проект. @@ -208,3 +212,46 @@ meta = ResqueJobTest.enqueue(id=2) ```ruby Resque.enqueue(ImageProcessingJob, id=2) ``` + +## Resque Scheduler + +Расписание для cron-like заданий должно храниться здесь `config/resque_schedule.yml` + +## Resque Retry + +В силу несовместимостей почти всех плагинов с resque-meta (unique основан на нём) - объявить задание перезапускаемым +в случае ошибки нужно ДО `unique` + +```ruby +class ResqueJobTest + include Resque::Integration + + retrys delay: 10, limit: 2 + unique +end +``` + +## Resque Multi Job Forks + +```yaml +workers: + 'high': + count: 1 + jobs_per_fork: 10 +``` + +## Gem Releasing + +1. должен быть настроен git remote upstream и должны быть права на push +1. git checkout master +2. git pull upstream master +3. правим версию гема в файле VERSION в корне гема. (читаем правила версионирования http://semver.org/) +4. bundle exec rake release + +## Contributing + +1. Fork it ( https://github.com/abak-press/resque-integration/fork ) +2. Create your feature branch (`git checkout -b my-new-feature`) +3. Commit your changes (`git commit -am 'Add some feature'`) +4. Push to the branch (`git push origin my-new-feature`) +5. Create a new Pull Request diff --git a/Rakefile b/Rakefile index 2995527..f6fc454 100644 --- a/Rakefile +++ b/Rakefile @@ -1 +1,7 @@ -require "bundler/gem_tasks" +begin + require 'bundler/setup' +rescue LoadError + puts 'You must `gem install bundler` and `bundle install` to run rake tasks' +end + +require 'apress/gems/rake_tasks' diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ + diff --git a/lib/resque/integration.rb b/lib/resque/integration.rb index 2f7dc02..cba9221 100644 --- a/lib/resque/integration.rb +++ b/lib/resque/integration.rb @@ -15,6 +15,11 @@ require 'resque/integration/hooks' require 'resque/integration/engine' +require 'resque/scheduler' +require 'resque/scheduler/tasks' + +require 'resque-retry' + require 'active_support/core_ext/module/attribute_accessors' module Resque @@ -79,6 +84,24 @@ def continuous def unique? false end + + # extend resque-retry + # + # options - Hash + # :limit - Integer (default: 2) + # :delay - Integer (default: 60) + # + # Returns nothing + def retrys(options = {}) + if unique? + raise '`retrys` should be declared higher in code than `unique`' + end + + extend Resque::Plugins::Retry + + @retry_limit = options.fetch(:limit, 2) + @retry_delay = options.fetch(:delay, 60) + end end end # module Integration -end # module Resque \ No newline at end of file +end # module Resque diff --git a/lib/resque/integration/configuration.rb b/lib/resque/integration/configuration.rb index 6b7c972..9775ffb 100644 --- a/lib/resque/integration/configuration.rb +++ b/lib/resque/integration/configuration.rb @@ -115,6 +115,11 @@ def verbosity (self['resque.verbosity'] || 0).to_i end + # Returns Resque log level + def log_level + (self['resque.log_level'] || 1).to_i + end + # Returns path to resque log file def log_file self['resque.log_file'] || ::Rails.root.join('log/resque.log').to_s @@ -136,6 +141,34 @@ def root self['resque.root'] || ::Rails.root.to_s end + # Путь до файла с расписание resque schedule + # + # Returns String + def schedule_file + self['resque.schedule_file'] || ::Rails.root.join('config', 'resque_schedule.yml') + end + + # Есть ли расписание у приложения? + # + # Returns boolean + def schedule_exists? + return @schedule_exists if defined?(@schedule_exists) + @schedule_exists = File.exist?(schedule_file) + end + + # Используется ли resque scheduler + # + # Returns boolean + def resque_scheduler? + value = self['resque.scheduler'] || true + + if value.is_a?(String) && %w(n no false off disabled).include?(value) + value = false + end + + value + end + # Returns maximum terminate timeout def terminate_timeout workers.map(&:stop_timeout).compact.max.to_i + 10 @@ -146,8 +179,6 @@ def env env = self['env'] || {} env[:INTERVAL] ||= interval - env[:VERBOSE] = '1' if verbosity == 1 - env[:VVERBOSE] = '1' if verbosity == 2 Hash[env.map { |k, v| [k, v.to_s] }] end @@ -182,4 +213,4 @@ def [](path) end end # class Configuration end # module Integration -end # module Resque \ No newline at end of file +end # module Resque diff --git a/lib/resque/integration/engine.rb b/lib/resque/integration/engine.rb index 23db22f..6744abd 100644 --- a/lib/resque/integration/engine.rb +++ b/lib/resque/integration/engine.rb @@ -12,7 +12,6 @@ class Engine < Rails::Engine rake_tasks do load 'resque/integration/tasks/hooks.rake' load 'resque/integration/tasks/resque.rake' - load 'resque/integration/tasks/supervisor.rake' # deprecated end # Читает конфиг-файлы config/resque.yml и config/resque.local.yml, @@ -35,6 +34,19 @@ class Engine < Rails::Engine end end + initializer 'resque-integration.logger' do + Resque.logger.level = Resque.config.log_level + + case Resque.config.verbosity + when 1 + Resque.logger.formatter = Resque::VerboseFormatter.new + when 2 + Resque.logger.formatter = Resque::VeryVerboseFormatter.new + else + Resque.logger.formatter = Resque::QuietFormatter.new + end + end + # Конфигурирование плагина resque-failed-job-mailer. # Данные берутся из конфига (см. выше) initializer 'resque-integration.failure_notifier' do @@ -53,4 +65,4 @@ class Engine < Rails::Engine end end end # class Engine -end # module Resque::Integration \ No newline at end of file +end # module Resque::Integration diff --git a/lib/resque/integration/god.erb b/lib/resque/integration/god.erb index 63adb9e..27a9a3b 100644 --- a/lib/resque/integration/god.erb +++ b/lib/resque/integration/god.erb @@ -52,4 +52,48 @@ God.terminate_timeout = <%= terminate_timeout.to_i %> # END OF WORKER CONFIGURATION end <% end %> -<% end %> \ No newline at end of file +<% end %> + +<% if resque_scheduler? %> + God.watch do |w| + w.dir = rails_root + w.name = 'resque-scheduler' + w.group = 'resque' + w.interval = 30.seconds + w.env = <%= Resque.config.env.merge(:RAILS_ENV => ::Rails.env, :BUNDLE_GEMFILE => "#{root}/Gemfile").stringify_keys! %> + w.start = "#{bundle} exec rake -f #{rails_root}/Rakefile environment resque:scheduler" + w.log = '<%= log_file %>' + w.stop_signal = 'QUIT' + w.stop_timeout = 60.seconds + + # determine the state on startup + w.transition(:init, { true => :up, false => :start }) do |on| + on.condition(:process_running) do |c| + c.running = true + end + end + + # determine when process has finished starting + w.transition([:start, :restart], :up) do |on| + on.condition(:process_running) do |c| + c.running = true + c.interval = 5.seconds + end + + # failsafe + on.condition(:tries) do |c| + c.times = 5 + c.transition = :start + c.interval = 5.seconds + end + end + + # start if process is not running + w.transition(:up, :start) do |on| + on.condition(:process_running) do |c| + c.running = false + end + end + # END OF SCHEDULE CONFIGURATION + end +<% end %> diff --git a/lib/resque/integration/tasks/hooks.rake b/lib/resque/integration/tasks/hooks.rake index d8f7ed2..5ec7107 100644 --- a/lib/resque/integration/tasks/hooks.rake +++ b/lib/resque/integration/tasks/hooks.rake @@ -30,11 +30,15 @@ namespace :resque do Resque.before_first_fork { Rails.cache.reset if Rails.cache.respond_to?(:reset) } # Красиво нарисуем название процесса - Resque.after_fork { |job| $0 = "resque-#{Resque::Version}: Processing #{job.queue}/#{job.payload['class']} since #{Time.now.to_s(:db)}" } + Resque.after_fork do |job| + $0 = "resque-#{Resque::Version}: Processing #{job.queue}/#{job.payload['class']} since #{Time.now.to_s(:db)}" + end # Support for resque-multi-job-forks - if ENV['JOBS_PER_FORK'] || ENV['MINUTES_PER_FORK'] - #require 'resque-multi-job-forks' + require 'resque-multi-job-forks' if ENV['JOBS_PER_FORK'] || ENV['MINUTES_PER_FORK'] + + if Resque.config.resque_scheduler? && Resque.config.schedule_exists? + Resque.schedule = YAML.load_file(Resque.config.schedule_file) end end end diff --git a/lib/resque/integration/tasks/supervisor.rake b/lib/resque/integration/tasks/supervisor.rake deleted file mode 100644 index 6b67e12..0000000 --- a/lib/resque/integration/tasks/supervisor.rake +++ /dev/null @@ -1,14 +0,0 @@ -# coding: utf-8 - -namespace :resque do - namespace :supervisor do - desc 'Starts supervisor process (no-op)' - task :start - - desc 'Stop supervisor process (no-op)' - task :stop - - desc 'Restart supervisor process (no-op)' - task :restart - end -end \ No newline at end of file diff --git a/lib/resque/integration/unique.rb b/lib/resque/integration/unique.rb index 96565c2..7f53e44 100644 --- a/lib/resque/integration/unique.rb +++ b/lib/resque/integration/unique.rb @@ -5,7 +5,7 @@ require 'active_support/core_ext/module/aliasing' require 'resque/plugins/lock' -require 'resque/plugins/progress' +silence_warnings { require 'resque/plugins/progress' } # suppress Resque::Helpers warn module Resque module Integration @@ -42,6 +42,18 @@ def unique? true end + # Метод вызывает resque-scheduler чтобы поставить задание в текущую очередь + def scheduled(queue, klass, *args) + klass.constantize.enqueue(*args) + end + + # Метод вызывает resque-retry когда ставить отложенное задание + # здесь мы убираем meta_id из аргументов + def retry_args(*args) + args.shift + args + end + # Get or set proc returning unique arguments def lock_on(&block) if block_given? @@ -164,4 +176,4 @@ def obj_to_string(obj) end # module ClassMethods end # module Unique end # module Integration -end # module Resque \ No newline at end of file +end # module Resque diff --git a/resque-integration.gemspec b/resque-integration.gemspec index ec6c63f..ea2e683 100644 --- a/resque-integration.gemspec +++ b/resque-integration.gemspec @@ -16,24 +16,26 @@ Gem::Specification.new do |gem| gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) gem.require_paths = %w(lib) - gem.add_runtime_dependency 'resque', '= 1.23.0' + gem.add_runtime_dependency 'resque', '= 1.25.2' gem.add_runtime_dependency 'railties', '>= 3.0.0' gem.add_runtime_dependency 'resque-rails', '>= 1.0.1' gem.add_runtime_dependency 'resque-ensure-connected', '>= 0.2.0' # reconnect after fork gem.add_runtime_dependency 'resque-lock', '~> 1.1.0' gem.add_runtime_dependency 'resque-meta', '>= 2.0.0' gem.add_runtime_dependency 'resque-progress', '~> 1.0.1' - #gem.add_runtime_dependency 'resque-multi-job-forks', '~> 0.3.4' + gem.add_runtime_dependency 'resque-multi-job-forks', '~> 0.4.2' gem.add_runtime_dependency 'resque-failed-job-mailer', '~> 0.0.3' + gem.add_runtime_dependency 'resque-scheduler', '~> 3.0' + gem.add_runtime_dependency 'resque-retry', '~> 1.3' gem.add_runtime_dependency 'god', '~> 0.13.4' gem.add_runtime_dependency 'multi_json' gem.add_runtime_dependency 'rake' gem.add_runtime_dependency 'sinatra' - gem.add_development_dependency 'rake' gem.add_development_dependency 'bundler' - gem.add_development_dependency 'rspec' + gem.add_development_dependency 'rspec', '~> 2.14' gem.add_development_dependency 'simplecov' gem.add_development_dependency 'mock_redis' + gem.add_development_dependency 'apress-gems', '>= 0.0.4' end diff --git a/spec/resque/integration/configuration_spec.rb b/spec/resque/integration/configuration_spec.rb index 0db6816..d61f071 100644 --- a/spec/resque/integration/configuration_spec.rb +++ b/spec/resque/integration/configuration_spec.rb @@ -2,6 +2,47 @@ require 'spec_helper' +describe Resque::Integration::Configuration do + let(:config) do + File.stub(:exists? => true) + File.stub(:read) + YAML.stub(:load => config_yaml) + described_class.new('path/to/config.yml') + end + + let(:config_yaml) { {} } + + describe '#schedule_file' do + let(:config_yaml) { {'resque' => {'schedule_file' => 'schedule.yml'}} } + + it { expect(config.schedule_file).to eq 'schedule.yml' } + end + + describe '#log_level' do + context 'when default' do + it { expect(config.log_level).to eq 1 } + end + + context 'when defined' do + let(:config_yaml) { {'resque' => {'log_level' => 2}} } + + it { expect(config.log_level).to eq 2 } + end + end + + describe '#resque_scheduler?' do + context 'when default' do + it { expect(config.resque_scheduler?).to be_true } + end + + context 'when defined' do + let(:config_yaml) { {'resque' => {'scheduler' => 'no'}} } + + it { expect(config.resque_scheduler?).to be_false } + end + end +end + describe Resque::Integration::Configuration::Notifier do context 'when NilClass given as config' do subject(:config) { described_class::new(nil) } @@ -82,4 +123,4 @@ its([:MINUTES_PER_FORK]) { should eq '5' } its([:VAR]) { should eq '2' } end -end \ No newline at end of file +end