From cfb539079f1bf2f07beda152d6264248f6e531ef Mon Sep 17 00:00:00 2001 From: Leenday Date: Fri, 6 Jan 2023 13:24:44 +0600 Subject: [PATCH 1/5] sidekiq configuration --- Gemfile | 1 + Gemfile.lock | 9 +++++++++ config/initializers/sidekiq.rb | 7 +++++++ config/sidekiq.yml | 4 ++++ docker-compose.yml | 14 ++++++++++++++ test/test_helper.rb | 1 + 6 files changed, 36 insertions(+) create mode 100644 config/initializers/sidekiq.rb create mode 100644 config/sidekiq.yml diff --git a/Gemfile b/Gemfile index 33a37e0..40d6fe7 100644 --- a/Gemfile +++ b/Gemfile @@ -31,6 +31,7 @@ gem 'js-routes' gem 'kaminari' gem 'ransack' gem 'responders' +gem 'sidekiq' gem 'simple_form' gem 'slim-rails' gem 'state_machines-activerecord' diff --git a/Gemfile.lock b/Gemfile.lock index 5d8127e..41202ec 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -83,6 +83,7 @@ GEM activesupport childprocess (4.1.0) concurrent-ruby (1.1.10) + connection_pool (2.3.0) crass (1.0.6) docile (1.4.0) erubi (1.11.0) @@ -191,6 +192,8 @@ GEM rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) + redis-client (0.11.2) + connection_pool regexp_parser (2.6.1) responders (3.0.1) actionpack (>= 5.0) @@ -225,6 +228,11 @@ GEM rexml (~> 3.2, >= 3.2.5) rubyzip (>= 1.2.2, < 3.0) websocket (~> 1.0) + sidekiq (7.0.2) + concurrent-ruby (< 2) + connection_pool (>= 2.3.0) + rack (>= 2.2.4) + redis-client (>= 0.11.0) simple_form (5.1.0) actionpack (>= 5.2) activemodel (>= 5.2) @@ -309,6 +317,7 @@ DEPENDENCIES rubocop sass-rails (>= 6) selenium-webdriver + sidekiq simple_form simplecov slim-rails diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb new file mode 100644 index 0000000..b061805 --- /dev/null +++ b/config/initializers/sidekiq.rb @@ -0,0 +1,7 @@ +Sidekiq.configure_server do |config| + config.redis = { url: ENV['REDIS_URL'] } +end + +Sidekiq.configure_client do |config| + config.redis = { url: ENV['REDIS_URL'] } +end diff --git a/config/sidekiq.yml b/config/sidekiq.yml new file mode 100644 index 0000000..c45a5a3 --- /dev/null +++ b/config/sidekiq.yml @@ -0,0 +1,4 @@ +:concurrency: 5 +:verbose: true +:queues: + - default diff --git a/docker-compose.yml b/docker-compose.yml index 1992957..917bd1a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,8 @@ services: - 3332:3332 depends_on: - db + - redis + - sidekiq environment: &web-environment BUNDLE_PATH: /bundle_cache @@ -32,6 +34,8 @@ services: MAILER_PORT: fake MAILER_DOMAIN: fake MAILER_AUTHENTICATION: fake + + REDIS_URL: redis://redis command: bundle exec rails s -b '0.0.0.0' -p 3330 db: @@ -42,5 +46,15 @@ services: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres + redis: + image: redis:5.0.9-alpine + + sidekiq: + build: . + command: bundle exec sidekiq -C /task_manager/config/sidekiq.yml + environment: *web-environment + volumes: *web-volumes + depends_on: + - redis volumes: bundle_cache: diff --git a/test/test_helper.rb b/test/test_helper.rb index 0c03942..dab70e3 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -4,6 +4,7 @@ ENV['RAILS_ENV'] ||= 'test' require_relative '../config/environment' require 'rails/test_help' +require 'sidekiq/testing' class ActiveSupport::TestCase include ActionMailer::TestHelper From 62fc78ad46a5ea55d63736d8c30b628e5acaa7f9 Mon Sep 17 00:00:00 2001 From: Leenday Date: Fri, 6 Jan 2023 13:30:58 +0600 Subject: [PATCH 2/5] adds mailers queue and usage of #deliver_later to mailer --- app/controllers/api/v1/tasks_controller.rb | 6 +++--- config/application.rb | 1 + config/sidekiq.yml | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/controllers/api/v1/tasks_controller.rb b/app/controllers/api/v1/tasks_controller.rb index df25aa7..8bd4480 100644 --- a/app/controllers/api/v1/tasks_controller.rb +++ b/app/controllers/api/v1/tasks_controller.rb @@ -18,20 +18,20 @@ def index def create params['task']['author_id'] = current_user.id if params.dig('task', 'author_id').nil? task = current_user.my_tasks.new(task_params) - UserMailer.with({ user: current_user, task: task }).task_created.deliver_now if task.save + UserMailer.with({ user: current_user, task: task }).task_created.deliver_later if task.save respond_with(task, serializer: TaskSerializer, location: nil) end def update task = Task.find(params[:id]) - UserMailer.with({ task: task }).task_updated.deliver_now if task.update(task_params) + UserMailer.with({ task: task }).task_updated.deliver_later if task.update(task_params) respond_with(task, serializer: TaskSerializer) end def destroy task = Task.find(params[:id]) - UserMailer.with({ task: task }).task_deleted.deliver_now if task.destroy + UserMailer.with({ task: task }).task_deleted.deliver_later if task.destroy respond_with(task) end diff --git a/config/application.rb b/config/application.rb index 5ca8e76..7275102 100644 --- a/config/application.rb +++ b/config/application.rb @@ -12,6 +12,7 @@ class Application < Rails::Application config.load_defaults 6.0 config.assets.paths << Rails.root.join('node_modules') + config.active_job.queue_adapter = :sidekiq # Settings in config/environments/* take precedence over those specified here. # Application configuration can go into files in config/initializers # -- all .rb files in that directory are automatically loaded after loading diff --git a/config/sidekiq.yml b/config/sidekiq.yml index c45a5a3..20aa386 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -2,3 +2,4 @@ :verbose: true :queues: - default + - mailers From 00cc48d7050b7b1fc4e3f17cbe25af2d7606c9f1 Mon Sep 17 00:00:00 2001 From: Leenday Date: Fri, 6 Jan 2023 14:22:05 +0600 Subject: [PATCH 3/5] add sidekiq-web and update redis --- Gemfile | 1 + Gemfile.lock | 3 +++ config/initializers/sidekiq.rb | 2 ++ config/routes.rb | 1 + docker-compose.yml | 2 +- 5 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 40d6fe7..bfd6ea4 100644 --- a/Gemfile +++ b/Gemfile @@ -32,6 +32,7 @@ gem 'kaminari' gem 'ransack' gem 'responders' gem 'sidekiq' +gem 'sidekiq-failures' gem 'simple_form' gem 'slim-rails' gem 'state_machines-activerecord' diff --git a/Gemfile.lock b/Gemfile.lock index 41202ec..3234873 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -233,6 +233,8 @@ GEM connection_pool (>= 2.3.0) rack (>= 2.2.4) redis-client (>= 0.11.0) + sidekiq-failures (1.0.4) + sidekiq (>= 4.0.0) simple_form (5.1.0) actionpack (>= 5.2) activemodel (>= 5.2) @@ -318,6 +320,7 @@ DEPENDENCIES sass-rails (>= 6) selenium-webdriver sidekiq + sidekiq-failures simple_form simplecov slim-rails diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index b061805..4abb475 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -1,3 +1,5 @@ +require 'sidekiq/web' + Sidekiq.configure_server do |config| config.redis = { url: ENV['REDIS_URL'] } end diff --git a/config/routes.rb b/config/routes.rb index 520db78..31c23ab 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,5 +1,6 @@ Rails.application.routes.draw do mount LetterOpenerWeb::Engine, at: '/letter_opener' if Rails.env.development? + mount Sidekiq::Web => '/admin/sidekiq' root to: 'web/boards#show' scope module: :web do diff --git a/docker-compose.yml b/docker-compose.yml index 917bd1a..5b60413 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,7 +47,7 @@ services: POSTGRES_PASSWORD: postgres redis: - image: redis:5.0.9-alpine + image: redis:7.0.7-alpine sidekiq: build: . From 4935bb0ca45b1fa9b3ce9f8a5e1857c6a7ff3ba9 Mon Sep 17 00:00:00 2001 From: Leenday Date: Fri, 6 Jan 2023 16:09:43 +0600 Subject: [PATCH 4/5] add sidekiq-throttled, workers and reworks tests --- Gemfile | 3 ++- Gemfile.lock | 22 +++++++++++-------- app/controllers/api/v1/tasks_controller.rb | 7 +++--- app/jobs/application_worker.rb | 4 ++++ .../send_task_create_notification_worker.rb | 11 ++++++++++ .../send_task_delete_notification_worker.rb | 11 ++++++++++ .../send_task_update_notification_worker.rb | 11 ++++++++++ config/initializers/sidekiq.rb | 5 +++++ test/test_helper.rb | 1 + 9 files changed, 62 insertions(+), 13 deletions(-) create mode 100644 app/jobs/application_worker.rb create mode 100644 app/jobs/send_task_create_notification_worker.rb create mode 100644 app/jobs/send_task_delete_notification_worker.rb create mode 100644 app/jobs/send_task_update_notification_worker.rb diff --git a/Gemfile b/Gemfile index bfd6ea4..34d7c7f 100644 --- a/Gemfile +++ b/Gemfile @@ -31,8 +31,9 @@ gem 'js-routes' gem 'kaminari' gem 'ransack' gem 'responders' -gem 'sidekiq' +gem 'sidekiq', '~> 6.5.0' gem 'sidekiq-failures' +gem 'sidekiq-throttled' gem 'simple_form' gem 'slim-rails' gem 'state_machines-activerecord' diff --git a/Gemfile.lock b/Gemfile.lock index 3234873..2706fe8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -152,7 +152,7 @@ GEM puma (4.3.12) nio4r (~> 2.0) racc (1.6.0) - rack (2.2.4) + rack (2.2.5) rack-proxy (0.7.4) rack rack-test (2.0.2) @@ -192,8 +192,8 @@ GEM rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) - redis-client (0.11.2) - connection_pool + redis (4.8.0) + redis-prescription (1.0.0) regexp_parser (2.6.1) responders (3.0.1) actionpack (>= 5.0) @@ -228,13 +228,16 @@ GEM rexml (~> 3.2, >= 3.2.5) rubyzip (>= 1.2.2, < 3.0) websocket (~> 1.0) - sidekiq (7.0.2) - concurrent-ruby (< 2) - connection_pool (>= 2.3.0) - rack (>= 2.2.4) - redis-client (>= 0.11.0) + sidekiq (6.5.8) + connection_pool (>= 2.2.5, < 3) + rack (~> 2.0) + redis (>= 4.5.0, < 5) sidekiq-failures (1.0.4) sidekiq (>= 4.0.0) + sidekiq-throttled (0.17.0) + concurrent-ruby + redis-prescription + sidekiq (>= 6.4) simple_form (5.1.0) actionpack (>= 5.2) activemodel (>= 5.2) @@ -319,8 +322,9 @@ DEPENDENCIES rubocop sass-rails (>= 6) selenium-webdriver - sidekiq + sidekiq (~> 6.5.0) sidekiq-failures + sidekiq-throttled simple_form simplecov slim-rails diff --git a/app/controllers/api/v1/tasks_controller.rb b/app/controllers/api/v1/tasks_controller.rb index 8bd4480..fecd8e2 100644 --- a/app/controllers/api/v1/tasks_controller.rb +++ b/app/controllers/api/v1/tasks_controller.rb @@ -18,20 +18,21 @@ def index def create params['task']['author_id'] = current_user.id if params.dig('task', 'author_id').nil? task = current_user.my_tasks.new(task_params) - UserMailer.with({ user: current_user, task: task }).task_created.deliver_later if task.save + SendTaskCreateNotificationWorker.perform_async(task.id) if task.save respond_with(task, serializer: TaskSerializer, location: nil) end def update task = Task.find(params[:id]) - UserMailer.with({ task: task }).task_updated.deliver_later if task.update(task_params) + SendTaskUpdateNotificationWorker.perform_async(task.id) if task.update(task_params) respond_with(task, serializer: TaskSerializer) end def destroy task = Task.find(params[:id]) - UserMailer.with({ task: task }).task_deleted.deliver_later if task.destroy + SendTaskDeleteNotificationWorker.perform_async(task.id) + task.destroy respond_with(task) end diff --git a/app/jobs/application_worker.rb b/app/jobs/application_worker.rb new file mode 100644 index 0000000..2a35cdd --- /dev/null +++ b/app/jobs/application_worker.rb @@ -0,0 +1,4 @@ +class ApplicationWorker + include Sidekiq::Worker + include Sidekiq::Throttled::Worker +end diff --git a/app/jobs/send_task_create_notification_worker.rb b/app/jobs/send_task_create_notification_worker.rb new file mode 100644 index 0000000..628dd30 --- /dev/null +++ b/app/jobs/send_task_create_notification_worker.rb @@ -0,0 +1,11 @@ +class SendTaskCreateNotificationWorker < ApplicationWorker + sidekiq_options queue: :mailers + sidekiq_throttle_as :mailer + + def perform(task_id) + task = Task.find_by(id: task_id) + return if task.blank? + + UserMailer.with(user: task.author, task: task).task_created.deliver_now + end +end diff --git a/app/jobs/send_task_delete_notification_worker.rb b/app/jobs/send_task_delete_notification_worker.rb new file mode 100644 index 0000000..06b64a0 --- /dev/null +++ b/app/jobs/send_task_delete_notification_worker.rb @@ -0,0 +1,11 @@ +class SendTaskDeleteNotificationWorker < ApplicationWorker + sidekiq_options queue: :mailers + sidekiq_throttle_as :mailer + + def perform(task_id) + task = Task.find_by(id: task_id) + return if task.blank? + + UserMailer.with(user: task.author, task: task).task_deleted.deliver_now + end +end diff --git a/app/jobs/send_task_update_notification_worker.rb b/app/jobs/send_task_update_notification_worker.rb new file mode 100644 index 0000000..191fbbb --- /dev/null +++ b/app/jobs/send_task_update_notification_worker.rb @@ -0,0 +1,11 @@ +class SendTaskUpdateNotificationWorker < ApplicationWorker + sidekiq_options queue: :mailers + sidekiq_throttle_as :mailer + + def perform(task_id) + task = Task.find_by(id: task_id) + return if task.blank? + + UserMailer.with(user: task.author, task: task).task_updated.deliver_now + end +end diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 4abb475..a79b633 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -1,4 +1,6 @@ require 'sidekiq/web' +require 'sidekiq/throttled' +require 'sidekiq/throttled/web' Sidekiq.configure_server do |config| config.redis = { url: ENV['REDIS_URL'] } @@ -7,3 +9,6 @@ Sidekiq.configure_client do |config| config.redis = { url: ENV['REDIS_URL'] } end + +Sidekiq::Throttled.setup! +Sidekiq::Throttled::Registry.add(:mailer, { threshold: { limit: 1, period: 5.seconds } }) diff --git a/test/test_helper.rb b/test/test_helper.rb index dab70e3..582205f 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -5,6 +5,7 @@ require_relative '../config/environment' require 'rails/test_help' require 'sidekiq/testing' +Sidekiq::Testing.inline! class ActiveSupport::TestCase include ActionMailer::TestHelper From 264aa3227d306736246f62023ca3304c8dbb8176 Mon Sep 17 00:00:00 2001 From: Leenday Date: Fri, 6 Jan 2023 16:23:18 +0600 Subject: [PATCH 5/5] adds a sidekiq-unique-jobs gem and configuration for it --- Gemfile | 1 + Gemfile.lock | 10 ++++++++++ app/jobs/send_task_update_notification_worker.rb | 1 + config/initializers/sidekiq.rb | 15 +++++++++++++++ 4 files changed, 27 insertions(+) diff --git a/Gemfile b/Gemfile index 34d7c7f..1bb5d3e 100644 --- a/Gemfile +++ b/Gemfile @@ -34,6 +34,7 @@ gem 'responders' gem 'sidekiq', '~> 6.5.0' gem 'sidekiq-failures' gem 'sidekiq-throttled' +gem 'sidekiq-unique-jobs' gem 'simple_form' gem 'slim-rails' gem 'state_machines-activerecord' diff --git a/Gemfile.lock b/Gemfile.lock index 2706fe8..c3a088f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -68,6 +68,9 @@ GEM bindex (0.8.1) bootsnap (1.15.0) msgpack (~> 1.2) + brpoplpush-redis_script (0.1.3) + concurrent-ruby (~> 1.0, >= 1.0.5) + redis (>= 1.0, < 6) builder (3.2.4) byebug (11.1.3) capybara (3.38.0) @@ -238,6 +241,12 @@ GEM concurrent-ruby redis-prescription sidekiq (>= 6.4) + sidekiq-unique-jobs (7.1.29) + brpoplpush-redis_script (> 0.1.1, <= 2.0.0) + concurrent-ruby (~> 1.0, >= 1.0.5) + redis (< 5.0) + sidekiq (>= 5.0, < 7.0) + thor (>= 0.20, < 3.0) simple_form (5.1.0) actionpack (>= 5.2) activemodel (>= 5.2) @@ -325,6 +334,7 @@ DEPENDENCIES sidekiq (~> 6.5.0) sidekiq-failures sidekiq-throttled + sidekiq-unique-jobs simple_form simplecov slim-rails diff --git a/app/jobs/send_task_update_notification_worker.rb b/app/jobs/send_task_update_notification_worker.rb index 191fbbb..c135403 100644 --- a/app/jobs/send_task_update_notification_worker.rb +++ b/app/jobs/send_task_update_notification_worker.rb @@ -1,5 +1,6 @@ class SendTaskUpdateNotificationWorker < ApplicationWorker sidekiq_options queue: :mailers + sidekiq_options lock: :until_and_while_executing, on_conflict: { client: :log, server: :reject } sidekiq_throttle_as :mailer def perform(task_id) diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index a79b633..f872a3d 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -1,13 +1,28 @@ require 'sidekiq/web' require 'sidekiq/throttled' require 'sidekiq/throttled/web' +require 'sidekiq_unique_jobs/web' Sidekiq.configure_server do |config| config.redis = { url: ENV['REDIS_URL'] } + + config.client_middleware do |chain| + chain.add SidekiqUniqueJobs::Middleware::Client + end + + config.server_middleware do |chain| + chain.add SidekiqUniqueJobs::Middleware::Server + end + + SidekiqUniqueJobs::Server.configure(config) end Sidekiq.configure_client do |config| config.redis = { url: ENV['REDIS_URL'] } + + config.client_middleware do |chain| + chain.add SidekiqUniqueJobs::Middleware::Client + end end Sidekiq::Throttled.setup!