From 80a12db1913c8ce568a6d6464bec75bcfd4e232b Mon Sep 17 00:00:00 2001 From: Maikel Linke Date: Tue, 31 Mar 2026 11:20:23 +1100 Subject: [PATCH 1/5] Move database clean from cron to Sidekiq scheduler After moving the remaining tasks from schedule.rb to sidekiq.yml, we can remove whenever and won't rely on cron any more. That will simplify the setup and migration to a new server. --- app/jobs/rake_job.rb | 13 +++++++++++++ config/schedule.rb | 4 ---- config/sidekiq.yml | 3 +++ spec/jobs/rake_job_spec.rb | 18 ++++++++++++++++++ 4 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 app/jobs/rake_job.rb create mode 100644 spec/jobs/rake_job_spec.rb diff --git a/app/jobs/rake_job.rb b/app/jobs/rake_job.rb new file mode 100644 index 0000000000..f522abd0a0 --- /dev/null +++ b/app/jobs/rake_job.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +# Executes a rake task +class RakeJob < ApplicationJob + def perform(task_string) + Rails.application.load_tasks if Rake::Task.tasks.empty? + + Rake.application.invoke_task(task_string) + ensure + name, _args = Rake.application.parse_task_string(task_string) + Rake::Task[name].reenable + end +end diff --git a/config/schedule.rb b/config/schedule.rb index 20c240f5ff..a07f4d9972 100644 --- a/config/schedule.rb +++ b/config/schedule.rb @@ -11,10 +11,6 @@ env "MAILTO", ENV["SCHEDULE_NOTIFICATIONS"] if ENV["SCHEDULE_NOTIFICATIONS"] # If we use -e with a file containing specs, rspec interprets it and filters out our examples job_type :run_file, "cd :path; :environment_variable=:environment bundle exec script/rails runner :task :output" -every 1.month, at: '4:30am' do - rake 'ofn:data:remove_transient_data' -end - every 1.day, at: '2:45am' do rake 'db2fog:clean' if ENV['S3_BACKUPS_BUCKET'] end diff --git a/config/sidekiq.yml b/config/sidekiq.yml index bcd4e31909..a2de4a4f97 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -19,3 +19,6 @@ every: "5m" OrderCycleClosingJob: every: "5m" + RakeJob: + args: ["ofn:data:remove_transient_data"] + cron: "30 4 1 * *" diff --git a/spec/jobs/rake_job_spec.rb b/spec/jobs/rake_job_spec.rb new file mode 100644 index 0000000000..9430c116d4 --- /dev/null +++ b/spec/jobs/rake_job_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require "tasks/data/remove_transient_data" + +RSpec.describe RakeJob do + let(:task_string) { "ofn:data:remove_transient_data" } + + it "calls the removal service" do + expect(RemoveTransientData).to receive(:new).and_call_original + RakeJob.perform_now(task_string) + end + + it "can be called several times" do + expect(RemoveTransientData).to receive(:new).twice.and_call_original + RakeJob.perform_now(task_string) + RakeJob.perform_now(task_string) + end +end From b61f6ab444def655f838eb3fb45a19ef01bd4976 Mon Sep 17 00:00:00 2001 From: Maikel Linke Date: Tue, 31 Mar 2026 14:53:26 +1100 Subject: [PATCH 2/5] Schedule all jobs with Sidekiq --- config/initializers/sidekiq.rb | 9 +++++++++ config/schedule.rb | 8 -------- config/sidekiq.yml | 20 +++++-------------- config/sidekiq_scheduler.yml | 36 ++++++++++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 23 deletions(-) create mode 100644 config/sidekiq_scheduler.yml diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 0628c9e409..25bf4bb557 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -7,6 +7,15 @@ redis_connection_settings = { Sidekiq.configure_server do |config| config.redis = redis_connection_settings + config.on(:startup) do + # Load schedule file similar to sidekiq/cli.rb loading the main config. + path = File.expand_path("../sidekiq_scheduler.yml", __dir__) + erb = ERB.new(File.read(path), trim_mode: "-") + + Sidekiq.schedule = + YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) + SidekiqScheduler::Scheduler.instance.reload_schedule! + end end Sidekiq.configure_client do |config| diff --git a/config/schedule.rb b/config/schedule.rb index a07f4d9972..5cd81427fc 100644 --- a/config/schedule.rb +++ b/config/schedule.rb @@ -10,11 +10,3 @@ env "MAILTO", ENV["SCHEDULE_NOTIFICATIONS"] if ENV["SCHEDULE_NOTIFICATIONS"] # If we use -e with a file containing specs, rspec interprets it and filters out our examples job_type :run_file, "cd :path; :environment_variable=:environment bundle exec script/rails runner :task :output" - -every 1.day, at: '2:45am' do - rake 'db2fog:clean' if ENV['S3_BACKUPS_BUCKET'] -end - -every 4.hours do - rake 'db2fog:backup' if ENV['S3_BACKUPS_BUCKET'] -end diff --git a/config/sidekiq.yml b/config/sidekiq.yml index a2de4a4f97..3be6e166dc 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -7,18 +7,8 @@ - default - mailers -:scheduler: - :schedule: - HeartbeatJob: - every: ["5m", first_in: "0s"] - SubscriptionPlacementJob: - every: "5m" - SubscriptionConfirmJob: - every: "5m" - TriggerOrderCyclesToOpenJob: - every: "5m" - OrderCycleClosingJob: - every: "5m" - RakeJob: - args: ["ofn:data:remove_transient_data"] - cron: "30 4 1 * *" +# This config is loaded by sidekiq before dotenv is loading our server config. +# Therefore we load the schedule later. See: +# +# - config/initializers/sidekiq.rb +# - config/sidekiq_scheduler.yml diff --git a/config/sidekiq_scheduler.yml b/config/sidekiq_scheduler.yml new file mode 100644 index 0000000000..29ef3a7e76 --- /dev/null +++ b/config/sidekiq_scheduler.yml @@ -0,0 +1,36 @@ +# Configure sidekiq-scheduler to run jobs. +# +# - https://github.com/sidekiq-scheduler/sidekiq-scheduler +# +# > Note that every and interval count from when the Sidekiq process (re)starts. +# > So every: '48h' will never run if the Sidekiq process is restarted daily, +# > for example. You can do every: ['48h', first_in: '0s'] to make the job run +# > immediately after a restart, and then have the worker check when it was +# > last run. +# +# Therefore, we use `cron` for jobs that should run at certain times like backups. +HeartbeatJob: + every: ["5m", first_in: "0s"] +SubscriptionPlacementJob: + every: "5m" +SubscriptionConfirmJob: + every: "5m" +TriggerOrderCyclesToOpenJob: + every: "5m" +OrderCycleClosingJob: + every: "5m" + +backup: + class: "RakeJob" + args: ["db2fog:backup"] + cron: "0 */4 * * *" # every 4 hours + enabled: <%= ENV.fetch("S3_BACKUPS_BUCKET", false) && true %> +backup_clean: + class: "RakeJob" + args: ["db2fog:clean"] + cron: "45 2 * * *" # every day at 2:45am + enabled: <%= ENV.fetch("S3_BACKUPS_BUCKET", false) && true %> +ofn_clean: + class: "RakeJob" + args: ["ofn:data:remove_transient_data"] + cron: "30 4 1 * *" # every month on the first at 4:30am From 60edcada2c2a6e9381ee8b5a70e36c3710593b70 Mon Sep 17 00:00:00 2001 From: Maikel Linke Date: Tue, 31 Mar 2026 14:56:41 +1100 Subject: [PATCH 3/5] Remove whenever config --- .env | 6 ------ config/schedule.rb | 12 ------------ 2 files changed, 18 deletions(-) delete mode 100644 config/schedule.rb diff --git a/.env b/.env index 3dc380b571..5f791c9f76 100644 --- a/.env +++ b/.env @@ -21,12 +21,6 @@ CHECKOUT_ZONE="Australia" # Find currency codes at http://en.wikipedia.org/wiki/ISO_4217. CURRENCY="AUD" -# The whenever gem can set the `MAILTO` variable for our cron jobs. -# You can define an email address to notify if any job outputs something. -# But you need a working mail server setup so that the message is delivered. -# See: config/schedule.rb -# SCHEDULE_NOTIFICATIONS="admin@example.com" - # Mail settings MAIL_HOST="example.com" MAIL_DOMAIN="example.com" diff --git a/config/schedule.rb b/config/schedule.rb deleted file mode 100644 index 5cd81427fc..0000000000 --- a/config/schedule.rb +++ /dev/null @@ -1,12 +0,0 @@ -# Force manual loading of rails application to get all env variables from dotenv-rails when running whenever cmd -require File.expand_path('../environment', __FILE__) - -require 'whenever' -require 'yaml' - -# Learn more: http://github.com/javan/whenever - -env "MAILTO", ENV["SCHEDULE_NOTIFICATIONS"] if ENV["SCHEDULE_NOTIFICATIONS"] - -# If we use -e with a file containing specs, rspec interprets it and filters out our examples -job_type :run_file, "cd :path; :environment_variable=:environment bundle exec script/rails runner :task :output" From c74624cd57847ed9e564416409e8aa2d17f23df2 Mon Sep 17 00:00:00 2001 From: Maikel Linke Date: Tue, 31 Mar 2026 14:58:57 +1100 Subject: [PATCH 4/5] Remove unused gem whenever --- Gemfile | 2 -- Gemfile.lock | 4 ---- 2 files changed, 6 deletions(-) diff --git a/Gemfile b/Gemfile index 9fab20b216..2ec71d0b3c 100644 --- a/Gemfile +++ b/Gemfile @@ -118,8 +118,6 @@ gem 'immigrant' gem 'roo' # read spreadsheets gem 'spreadsheet_architect' # write spreadsheets -gem 'whenever', require: false - gem 'coffee-rails', '~> 5.0.0' gem 'angular_rails_csrf' diff --git a/Gemfile.lock b/Gemfile.lock index 81c0e591d4..484cfce62d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -245,7 +245,6 @@ GEM cgi (0.5.1) childprocess (5.0.0) choice (0.2.0) - chronic (0.10.2) coderay (1.1.3) coffee-rails (5.0.0) coffee-script (>= 2.2.0) @@ -968,8 +967,6 @@ GEM base64 websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) - whenever (1.1.2) - chronic (>= 0.6.3) xml-simple (1.1.8) xpath (3.2.0) nokogiri (~> 1.8) @@ -1127,7 +1124,6 @@ DEPENDENCIES web! web-console webmock - whenever wicked_pdf! wkhtmltopdf-binary! From bcecbf9a0fa94e3078f0a5ac2d9a91f4c14489c0 Mon Sep 17 00:00:00 2001 From: Maikel Linke Date: Wed, 1 Apr 2026 15:18:49 +1100 Subject: [PATCH 5/5] Require rake dependency to run it within jobs --- app/jobs/rake_job.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/jobs/rake_job.rb b/app/jobs/rake_job.rb index f522abd0a0..07f89b72e8 100644 --- a/app/jobs/rake_job.rb +++ b/app/jobs/rake_job.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require "rake" + # Executes a rake task class RakeJob < ApplicationJob def perform(task_string)