From eb846e27fd2398e7b23d227006fa0652837cecf3 Mon Sep 17 00:00:00 2001 From: Rohan Mitchell Date: Wed, 13 Apr 2016 15:53:53 +1000 Subject: [PATCH 1/4] Run a job queue heartbeat every 5 minutes --- app/jobs/heartbeat_job.rb | 5 ++++ .../spree/app_configuration_decorator.rb | 3 +++ config/schedule.rb | 8 ++++++ spec/jobs/heartbeat_job_spec.rb | 27 +++++++++++++++++++ 4 files changed, 43 insertions(+) create mode 100644 app/jobs/heartbeat_job.rb create mode 100644 spec/jobs/heartbeat_job_spec.rb diff --git a/app/jobs/heartbeat_job.rb b/app/jobs/heartbeat_job.rb new file mode 100644 index 0000000000..93e835905f --- /dev/null +++ b/app/jobs/heartbeat_job.rb @@ -0,0 +1,5 @@ +class HeartbeatJob + def perform + Spree::Config.last_job_queue_heartbeat_at = Time.now + end +end diff --git a/app/models/spree/app_configuration_decorator.rb b/app/models/spree/app_configuration_decorator.rb index fc7a8171cc..6ef1e7b848 100644 --- a/app/models/spree/app_configuration_decorator.rb +++ b/app/models/spree/app_configuration_decorator.rb @@ -20,4 +20,7 @@ Spree::AppConfiguration.class_eval do preference :account_invoices_monthly_rate, :decimal, default: 0 preference :account_invoices_monthly_cap, :decimal, default: 0 preference :account_invoices_tax_rate, :decimal, default: 0 + + # Monitoring + preference :last_job_queue_heartbeat_at, :string, default: nil end diff --git a/config/schedule.rb b/config/schedule.rb index 023382330a..6f8e7b582a 100644 --- a/config/schedule.rb +++ b/config/schedule.rb @@ -4,9 +4,13 @@ require 'whenever' env "MAILTO", "rohan@rohanmitchell.com" + # If we use -e with a file containing specs, rspec interprets it and filters out our examples job_type :run_file, "cd :path; :environment_variable=:environment bundle exec script/rails runner :task :output" +job_type :enqueue_job, "cd :path; :environment_variable=:environment bundle exec script/rails runner 'Delayed::Job.enqueue(:task.new, priority: :priority)' :output" + + every 1.hour do rake 'openfoodnetwork:cache:check_products_integrity' end @@ -23,6 +27,10 @@ every 4.hours do rake 'db2fog:backup' end +every 5.minutes do + enqueue_job 'QueueHeartbeatJob', priority: 0 +end + every 1.day, at: '1:00am' do rake 'openfoodnetwork:billing:update_account_invoices' end diff --git a/spec/jobs/heartbeat_job_spec.rb b/spec/jobs/heartbeat_job_spec.rb new file mode 100644 index 0000000000..3bafecd572 --- /dev/null +++ b/spec/jobs/heartbeat_job_spec.rb @@ -0,0 +1,27 @@ +require 'spec_helper' + +describe HeartbeatJob do + context "with time frozen" do + let(:run_time) { Time.zone.local(2016, 4, 13, 13, 0, 0) } + + before { Spree::Config.last_job_queue_heartbeat_at = nil } + + around do |example| + Timecop.freeze(run_time) { example.run } + end + + it "updates the last_job_queue_heartbeat_at config var" do + run_job + Time.parse(Spree::Config.last_job_queue_heartbeat_at).should == run_time + end + end + + + private + + def run_job + clear_jobs + Delayed::Job.enqueue HeartbeatJob.new + flush_jobs ignore_exceptions: false + end +end From 193580d5d9e5ec60b389fbbb31d77a31fe93e16c Mon Sep 17 00:00:00 2001 From: Rohan Mitchell Date: Wed, 13 Apr 2016 16:01:21 +1000 Subject: [PATCH 2/4] Enqueue jobs directly via SQL rather than loading full Rails stack --- config/schedule.rb | 5 ++-- script/enqueue | 61 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 3 deletions(-) create mode 100755 script/enqueue diff --git a/config/schedule.rb b/config/schedule.rb index 6f8e7b582a..8cc8d7d225 100644 --- a/config/schedule.rb +++ b/config/schedule.rb @@ -7,8 +7,7 @@ env "MAILTO", "rohan@rohanmitchell.com" # If we use -e with a file containing specs, rspec interprets it and filters out our examples job_type :run_file, "cd :path; :environment_variable=:environment bundle exec script/rails runner :task :output" - -job_type :enqueue_job, "cd :path; :environment_variable=:environment bundle exec script/rails runner 'Delayed::Job.enqueue(:task.new, priority: :priority)' :output" +job_type :enqueue_job, "cd :path; :environment_variable=:environment bundle exec script/enqueue :task :priority :output" every 1.hour do @@ -28,7 +27,7 @@ every 4.hours do end every 5.minutes do - enqueue_job 'QueueHeartbeatJob', priority: 0 + enqueue_job 'HeartbeatJob', priority: 0 end every 1.day, at: '1:00am' do diff --git a/script/enqueue b/script/enqueue new file mode 100755 index 0000000000..2071414e4e --- /dev/null +++ b/script/enqueue @@ -0,0 +1,61 @@ +#!/usr/bin/env ruby + +# Push a job onto the Delayed Job queue without booting the Rails stack +# Perfect for calling via cron. +# +# Use like this: +# +# ./script/enqueue +# ./script/enqueue Background::ImportJobs + +require 'erb' +require 'yaml' + +ENV["RAILS_ENV"] ||= "development" + +DATABASE_CONFIG = File.expand_path("../../config/database.yml", __FILE__) + +def psql + raise "Missing database.yml" unless File.exists?(DATABASE_CONFIG) + + file = File.read(DATABASE_CONFIG) + erb = ERB.new(file).result + env = ENV["RAILS_ENV"] + config = YAML.load(erb)[env] + + raise "Missing config for #{env} environment" unless config + + "psql".tap do |s| + s << " --host #{config['host']}" if config['host'] + s << " --user #{config['username']}" if config['username'] + s << " --port #{config['port']}" if config['port'] + s << " #{config['database']}" + end +end + +def enqueue_delayed_job(handler, priority=nil) + time = Time.now.utc.strftime("%Y-%m-%d %H:%M:%S") + priority ||= 50 + + sql = <<-SQL + INSERT INTO delayed_jobs ( + handler, + created_at, + updated_at, + run_at, + priority + ) VALUES ( + '--- !ruby/object:#{handler} {}\n', + '#{time}', + '#{time}', + '#{time}', + #{priority} + ); + SQL + + IO.popen(psql, "w") do |io| + io.write sql + end +end + +enqueue_delayed_job ARGV[0], ARGV[1] From 59b564c4bebf9ec5924cb92385ac81d3038a2a5b Mon Sep 17 00:00:00 2001 From: Rohan Mitchell Date: Wed, 13 Apr 2016 16:48:07 +1000 Subject: [PATCH 3/4] Report job queue status via API --- app/controllers/api/statuses_controller.rb | 17 +++++++++++ config/routes.rb | 4 +++ .../api/statuses_controller_spec.rb | 30 +++++++++++++++++++ 3 files changed, 51 insertions(+) create mode 100644 app/controllers/api/statuses_controller.rb create mode 100644 spec/controllers/api/statuses_controller_spec.rb diff --git a/app/controllers/api/statuses_controller.rb b/app/controllers/api/statuses_controller.rb new file mode 100644 index 0000000000..c8844b868b --- /dev/null +++ b/app/controllers/api/statuses_controller.rb @@ -0,0 +1,17 @@ +module Api + class StatusesController < BaseController + respond_to :json + + def job_queue + render json: {alive: job_queue_alive?} + end + + + private + + def job_queue_alive? + Spree::Config.last_job_queue_heartbeat_at.present? && + Time.parse(Spree::Config.last_job_queue_heartbeat_at) > 6.minutes.ago + end + end +end diff --git a/config/routes.rb b/config/routes.rb index 26d675838c..6293117a3f 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -143,6 +143,10 @@ Openfoodnetwork::Application.routes.draw do get :managed, on: :collection get :accessible, on: :collection end + + resource :status do + get :job_queue + end end namespace :open_food_network do diff --git a/spec/controllers/api/statuses_controller_spec.rb b/spec/controllers/api/statuses_controller_spec.rb new file mode 100644 index 0000000000..f2427efb79 --- /dev/null +++ b/spec/controllers/api/statuses_controller_spec.rb @@ -0,0 +1,30 @@ +require 'spec_helper' + +module Api + describe StatusesController do + render_views + + describe "job queue status" do + it "returns alive when up to date" do + Spree::Config.last_job_queue_heartbeat_at = Time.now + spree_get :job_queue + response.should be_success + response.body.should == {alive: true}.to_json + end + + it "returns dead otherwise" do + Spree::Config.last_job_queue_heartbeat_at = 10.minutes.ago + spree_get :job_queue + response.should be_success + response.body.should == {alive: false}.to_json + end + + it "returns dead when no heartbeat recorded" do + Spree::Config.last_job_queue_heartbeat_at = nil + spree_get :job_queue + response.should be_success + response.body.should == {alive: false}.to_json + end + end + end +end From 53e6d391e3fdbb6c76e1ededebb9ca4eea8204c4 Mon Sep 17 00:00:00 2001 From: Rohan Mitchell Date: Thu, 14 Apr 2016 09:40:55 +1000 Subject: [PATCH 4/4] Fix intermittent fail on insignificant result ordering --- .../open_food_network/order_cycle_management_report_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/lib/open_food_network/order_cycle_management_report_spec.rb b/spec/lib/open_food_network/order_cycle_management_report_spec.rb index 2ac752561e..063470f37a 100644 --- a/spec/lib/open_food_network/order_cycle_management_report_spec.rb +++ b/spec/lib/open_food_network/order_cycle_management_report_spec.rb @@ -92,7 +92,7 @@ module OpenFoodNetwork # payment2 = create(:payment, order: order2, payment_method: pm2) subject.stub(:params).and_return(payment_method_in: [pm1.id, pm3.id] ) - subject.filter(orders).should == [order1, order3] + subject.filter(orders).should match_array [order1, order3] end it "filters to a shipping method" do @@ -102,7 +102,7 @@ module OpenFoodNetwork order3 = create(:order, shipping_method: sm3) subject.stub(:params).and_return(shipping_method_in: [sm1.id, sm3.id]) - subject.filter(orders).should == [order1, order3] + subject.filter(orders).should match_array [order1, order3] end it "should do all the filters at once" do