diff --git a/lib/tasks/data/truncate_data.rake b/lib/tasks/data/truncate_data.rake index a815f6ea1f..65075a3ffb 100644 --- a/lib/tasks/data/truncate_data.rake +++ b/lib/tasks/data/truncate_data.rake @@ -1,81 +1,45 @@ +# frozen_string_literal: true + +require 'highline' +require 'tasks/data/truncate_data' + # This task can be used to significantly reduce the size of a database # This is used for example when loading live data into a staging server # This way the staging server is not overloaded with too much data +# +# This is also aimed at implementing data archiving. We assume data older than +# 2 years can be safely removed and restored from a backup. This gives room for +# hubs to do their tax declaration. +# +# It's a must to perform a backup right before executing this. Then, to allow +# for a later data recovery we need to keep track of the exact moment this rake +# task was executed. +# +# Execute this in production only when the instance users are sleeping to avoid any trouble. +# +# Example: +# +# $ bundle exec rake "ofn:data:truncate[24]" +# +# This will remove data older than 2 years (24 months). namespace :ofn do namespace :data do desc 'Truncate data' - task truncate: :environment do - guard_and_warn + task :truncate, [:months_to_keep] => :environment do |_task, args| + warn_with_confirmation - sql_delete_from " - spree_inventory_units #{where_order_id_in_orders_to_delete}" - sql_delete_from " - spree_inventory_units - where shipment_id in (select id from spree_shipments #{where_order_id_in_orders_to_delete})" - - truncate_adjustments - - sql_delete_from "spree_line_items #{where_order_id_in_orders_to_delete}" - sql_delete_from "spree_payments #{where_order_id_in_orders_to_delete}" - sql_delete_from "spree_shipments #{where_order_id_in_orders_to_delete}" - Spree::ReturnAuthorization.delete_all - - truncate_order_cycle_data - - sql_delete_from "proxy_orders #{where_oc_id_in_ocs_to_delete}" - - sql_delete_from "spree_orders #{where_oc_id_in_ocs_to_delete}" - sql_delete_from "order_cycle_schedules #{where_oc_id_in_ocs_to_delete}" - sql_delete_from "order_cycles #{where_ocs_to_delete}" - - Spree::TokenizedPermission.where("created_at < '#{date}'").delete_all - Spree::StateChange.delete_all - Spree::LogEntry.delete_all - sql_delete_from "sessions" + TruncateData.new(args.months_to_keep).call end - def sql_delete_from(sql) - ActiveRecord::Base.connection.execute("delete from #{sql}") - end + def warn_with_confirmation + message = <<-MSG.strip_heredoc + \n + <% highlighted_message = "This will permanently change DB contents. This is not meant to be run in production as it needs more thorough testing." %> + <%= color(highlighted_message, :blink, :on_red) %> + Are you sure you want to proceed? (y/N) + MSG - private - - def date - 3.months.ago - end - - def where_ocs_to_delete - "where orders_close_at < '#{date}'" - end - - def where_oc_id_in_ocs_to_delete - "where order_cycle_id in (select id from order_cycles #{where_ocs_to_delete} )" - end - - def where_order_id_in_orders_to_delete - "where order_id in (select id from spree_orders #{where_oc_id_in_ocs_to_delete})" - end - - def truncate_adjustments - sql_delete_from "spree_adjustments where source_type = 'Spree::Order' - and source_id in (select id from spree_orders #{where_oc_id_in_ocs_to_delete})" - sql_delete_from "spree_adjustments where source_type = 'Spree::Shipment' - and source_id in (select id from spree_shipments #{where_order_id_in_orders_to_delete})" - sql_delete_from "spree_adjustments where source_type = 'Spree::Payment' - and source_id in (select id from spree_payments #{where_order_id_in_orders_to_delete})" - sql_delete_from "spree_adjustments where source_type = 'Spree::LineItem' - and source_id in (select id from spree_line_items #{where_order_id_in_orders_to_delete})" - end - - def truncate_order_cycle_data - sql_delete_from "coordinator_fees #{where_oc_id_in_ocs_to_delete}" - sql_delete_from " - exchange_variants where exchange_id - in (select id from exchanges #{where_oc_id_in_ocs_to_delete})" - sql_delete_from " - exchange_fees where exchange_id - in (select id from exchanges #{where_oc_id_in_ocs_to_delete})" - sql_delete_from "exchanges #{where_oc_id_in_ocs_to_delete}" + exit unless HighLine.new.agree(message) { |question| question.default = "N" } end end end diff --git a/lib/tasks/data/truncate_data.rb b/lib/tasks/data/truncate_data.rb new file mode 100644 index 0000000000..34300ca1dc --- /dev/null +++ b/lib/tasks/data/truncate_data.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +class TruncateData + # This model lets us operate on the sessions DB table using ActiveRecord's + # methods within the scope of this service. This relies on the AR's + # convention where a Session model maps to a sessions table. + class Session < ActiveRecord::Base + end + + def initialize(months_to_keep = nil) + @date = (months_to_keep || 24).to_i.months.ago + end + + def call + logging do + truncate_inventory + truncate_adjustments + truncate_order_associations + truncate_order_cycle_data + + sql_delete_from "spree_orders #{where_oc_id_in_ocs_to_delete}" + + truncate_subscriptions + + sql_delete_from "order_cycles #{where_ocs_to_delete}" + + Spree::TokenizedPermission.where("created_at < '#{date}'").delete_all + + remove_transient_data + end + end + + private + + attr_reader :date + + def logging + Rails.logger.info("TruncateData started with truncation date #{date}") + yield + Rails.logger.info("TruncateData finished") + end + + def truncate_order_associations + sql_delete_from "spree_line_items #{where_order_id_in_orders_to_delete}" + sql_delete_from "spree_payments #{where_order_id_in_orders_to_delete}" + sql_delete_from "spree_shipments #{where_order_id_in_orders_to_delete}" + sql_delete_from "spree_return_authorizations #{where_order_id_in_orders_to_delete}" + end + + def remove_transient_data + Spree::StateChange.delete_all("created_at < '#{1.month.ago.to_date}'") + Spree::LogEntry.delete_all("created_at < '#{1.month.ago.to_date}'") + Session.delete_all("created_at < '#{2.weeks.ago.to_date}'") + end + + def truncate_subscriptions + sql_delete_from "order_cycle_schedules #{where_oc_id_in_ocs_to_delete}" + sql_delete_from "proxy_orders #{where_oc_id_in_ocs_to_delete}" + end + + def truncate_inventory + sql_delete_from " + spree_inventory_units #{where_order_id_in_orders_to_delete}" + sql_delete_from " + spree_inventory_units + where shipment_id in (select id from spree_shipments #{where_order_id_in_orders_to_delete})" + end + + def sql_delete_from(sql) + ActiveRecord::Base.connection.execute("DELETE FROM #{sql}") + end + + def where_order_id_in_orders_to_delete + "where order_id in (select id from spree_orders #{where_oc_id_in_ocs_to_delete})" + end + + def where_oc_id_in_ocs_to_delete + "where order_cycle_id in (select id from order_cycles #{where_ocs_to_delete} )" + end + + def where_ocs_to_delete + "where orders_close_at < '#{date}'" + end + + def truncate_adjustments + sql_delete_from "spree_adjustments where source_type = 'Spree::Order' + and source_id in (select id from spree_orders #{where_oc_id_in_ocs_to_delete})" + + sql_delete_from "spree_adjustments where source_type = 'Spree::Shipment' + and source_id in (select id from spree_shipments #{where_order_id_in_orders_to_delete})" + + sql_delete_from "spree_adjustments where source_type = 'Spree::Payment' + and source_id in (select id from spree_payments #{where_order_id_in_orders_to_delete})" + + sql_delete_from "spree_adjustments where source_type = 'Spree::LineItem' + and source_id in (select id from spree_line_items #{where_order_id_in_orders_to_delete})" + end + + def truncate_order_cycle_data + sql_delete_from "coordinator_fees #{where_oc_id_in_ocs_to_delete}" + sql_delete_from " + exchange_variants where exchange_id + in (select id from exchanges #{where_oc_id_in_ocs_to_delete})" + sql_delete_from " + exchange_fees where exchange_id + in (select id from exchanges #{where_oc_id_in_ocs_to_delete})" + sql_delete_from "exchanges #{where_oc_id_in_ocs_to_delete}" + end +end diff --git a/spec/lib/tasks/data/truncate_data_rake_spec.rb b/spec/lib/tasks/data/truncate_data_rake_spec.rb new file mode 100644 index 0000000000..fff40d5b1f --- /dev/null +++ b/spec/lib/tasks/data/truncate_data_rake_spec.rb @@ -0,0 +1,34 @@ +require 'spec_helper' +require 'rake' + +describe 'truncate_data.rake' do + describe ':truncate' do + context 'when months_to_keep is specified' do + it 'truncates order cycles closed earlier than months_to_keep months ago' do + Rake.application.rake_require 'tasks/data/truncate_data' + Rake::Task.define_task(:environment) + + highline = instance_double(HighLine, agree: true) + allow(HighLine).to receive(:new).and_return(highline) + + old_order_cycle = create( + :order_cycle, + orders_open_at: 7.months.ago, + orders_close_at: 7.months.ago + 1.day, + ) + create(:order, order_cycle: old_order_cycle) + recent_order_cycle = create( + :order_cycle, + orders_open_at: 1.months.ago, + orders_close_at: 1.months.ago + 1.day, + ) + create(:order, order_cycle: recent_order_cycle) + + months_to_keep = 6 + Rake.application.invoke_task "ofn:data:truncate[#{months_to_keep}]" + + expect(OrderCycle.all).to contain_exactly(recent_order_cycle) + end + end + end +end diff --git a/spec/lib/tasks/data/truncate_data_spec.rb b/spec/lib/tasks/data/truncate_data_spec.rb new file mode 100644 index 0000000000..850b9bb9a8 --- /dev/null +++ b/spec/lib/tasks/data/truncate_data_spec.rb @@ -0,0 +1,130 @@ +require 'spec_helper' +require 'tasks/data/truncate_data' + +describe TruncateData do + describe '#call' do + before do + allow(Spree::ReturnAuthorization).to receive(:delete_all) + allow(Spree::StateChange).to receive(:delete_all) + allow(Spree::LogEntry).to receive(:delete_all) + allow(TruncateData::Session).to receive(:delete_all) + allow(Rails.logger).to receive(:info) + end + + context 'when months_to_keep is not specified' do + it 'truncates order cycles closed earlier than 2 years ago' do + order_cycle = create( + :order_cycle, orders_open_at: 25.months.ago, orders_close_at: 25.months.ago + 1.day + ) + create(:order, order_cycle: order_cycle) + + TruncateData.new.call + + expect(OrderCycle.all).to be_empty + end + + it 'deletes state changes older than a month' do + TruncateData.new.call + + expect(Spree::StateChange) + .to have_received(:delete_all) + .with("created_at < '#{1.month.ago.to_date}'") + end + + it 'deletes log entries older than a month' do + TruncateData.new.call + + expect(Spree::LogEntry) + .to have_received(:delete_all) + .with("created_at < '#{1.month.ago.to_date}'") + end + + it 'deletes sessions older than two weeks' do + TruncateData.new.call + + expect(TruncateData::Session) + .to have_received(:delete_all) + .with("created_at < '#{2.weeks.ago.to_date}'") + end + end + + context 'when months_to_keep is nil' do + it 'truncates order cycles closed earlier than 2 years ago' do + order_cycle = create( + :order_cycle, orders_open_at: 25.months.ago, orders_close_at: 25.months.ago + 1.day + ) + create(:order, order_cycle: order_cycle) + + TruncateData.new(nil).call + + expect(OrderCycle.all).to be_empty + end + + it 'deletes state changes older than a month' do + TruncateData.new.call + + expect(Spree::StateChange) + .to have_received(:delete_all) + .with("created_at < '#{1.month.ago.to_date}'") + end + + it 'deletes log entries older than a month' do + TruncateData.new.call + + expect(Spree::LogEntry) + .to have_received(:delete_all) + .with("created_at < '#{1.month.ago.to_date}'") + end + + it 'deletes sessions older than two weeks' do + TruncateData.new.call + + expect(TruncateData::Session) + .to have_received(:delete_all) + .with("created_at < '#{2.weeks.ago.to_date}'") + end + end + + context 'when months_to_keep is specified' do + it 'truncates order cycles closed earlier than months_to_keep months ago' do + old_order_cycle = create( + :order_cycle, orders_open_at: 7.months.ago, orders_close_at: 7.months.ago + 1.day + ) + create(:order, order_cycle: old_order_cycle) + recent_order_cycle = create( + :order_cycle, orders_open_at: 1.months.ago, orders_close_at: 1.months.ago + 1.day + ) + create(:order, order_cycle: recent_order_cycle) + + TruncateData.new(6).call + + expect(OrderCycle.all).to contain_exactly(recent_order_cycle) + end + + it 'deletes state changes older than a month' do + TruncateData.new.call + + expect(Spree::StateChange) + .to have_received(:delete_all) + .with("created_at < '#{1.month.ago.to_date}'") + end + + it 'deletes log entries older than a month' do + TruncateData.new.call + + expect(Spree::LogEntry) + .to have_received(:delete_all) + .with("created_at < '#{1.month.ago.to_date}'") + end + + it 'deletes sessions older than two weeks' do + TruncateData.new.call + + expect(TruncateData::Session) + .to have_received(:delete_all) + .with("created_at < '#{2.weeks.ago.to_date}'") + end + end + end +end +