Merge pull request #4935 from coopdevs/data-archiving

Allow data archiving using the :truncate_data task

lib/tasks/data/truncate_data.rake

@@ -1,81 +1,45 @@
+# frozen_string_literal: true
+
+require 'highline'
+require 'tasks/data/truncate_data'
+
 # This task can be used to significantly reduce the size of a database
 #   This is used for example when loading live data into a staging server
 #   This way the staging server is not overloaded with too much data
+#
+# This is also aimed at implementing data archiving. We assume data older than
+# 2 years can be safely removed and restored from a backup. This gives room for
+# hubs to do their tax declaration.
+#
+# It's a must to perform a backup right before executing this. Then, to allow
+# for a later data recovery we need to keep track of the exact moment this rake
+# task was executed.
+#
+# Execute this in production only when the instance users are sleeping to avoid any trouble.
+#
+# Example:
+#
+# $ bundle exec rake "ofn:data:truncate[24]"
+#
+# This will remove data older than 2 years (24 months).
 namespace :ofn do
   namespace :data do
     desc 'Truncate data'
-    task truncate: :environment do
-      guard_and_warn
+    task :truncate, [:months_to_keep] => :environment do |_task, args|
+      warn_with_confirmation
 
-      sql_delete_from "
-        spree_inventory_units #{where_order_id_in_orders_to_delete}"
-      sql_delete_from "
-        spree_inventory_units
-        where shipment_id in (select id from spree_shipments #{where_order_id_in_orders_to_delete})"
-
-      truncate_adjustments
-
-      sql_delete_from "spree_line_items #{where_order_id_in_orders_to_delete}"
-      sql_delete_from "spree_payments #{where_order_id_in_orders_to_delete}"
-      sql_delete_from "spree_shipments #{where_order_id_in_orders_to_delete}"
-      Spree::ReturnAuthorization.delete_all
-
-      truncate_order_cycle_data
-
-      sql_delete_from "proxy_orders #{where_oc_id_in_ocs_to_delete}"
-
-      sql_delete_from "spree_orders #{where_oc_id_in_ocs_to_delete}"
-      sql_delete_from "order_cycle_schedules #{where_oc_id_in_ocs_to_delete}"
-      sql_delete_from "order_cycles #{where_ocs_to_delete}"
-
-      Spree::TokenizedPermission.where("created_at < '#{date}'").delete_all
-      Spree::StateChange.delete_all
-      Spree::LogEntry.delete_all
-      sql_delete_from "sessions"
+      TruncateData.new(args.months_to_keep).call
     end
 
-    def sql_delete_from(sql)
-      ActiveRecord::Base.connection.execute("delete from #{sql}")
-    end
+    def warn_with_confirmation
+      message = <<-MSG.strip_heredoc
+      \n
+      <% highlighted_message  = "This will permanently change DB contents. This is not meant to be run in production as it needs more thorough testing." %>
+      <%= color(highlighted_message, :blink, :on_red) %>
+      Are you sure you want to proceed? (y/N)
+      MSG
 
-    private
-
-    def date
-      3.months.ago
-    end
-
-    def where_ocs_to_delete
-      "where orders_close_at < '#{date}'"
-    end
-
-    def where_oc_id_in_ocs_to_delete
-      "where order_cycle_id in (select id from order_cycles #{where_ocs_to_delete} )"
-    end
-
-    def where_order_id_in_orders_to_delete
-      "where order_id in (select id from spree_orders #{where_oc_id_in_ocs_to_delete})"
-    end
-
-    def truncate_adjustments
-      sql_delete_from "spree_adjustments where source_type = 'Spree::Order'
-        and source_id in (select id from spree_orders #{where_oc_id_in_ocs_to_delete})"
-      sql_delete_from "spree_adjustments where source_type = 'Spree::Shipment'
-        and source_id in (select id from spree_shipments #{where_order_id_in_orders_to_delete})"
-      sql_delete_from "spree_adjustments where source_type = 'Spree::Payment'
-        and source_id in (select id from spree_payments #{where_order_id_in_orders_to_delete})"
-      sql_delete_from "spree_adjustments where source_type = 'Spree::LineItem'
-        and source_id in (select id from spree_line_items #{where_order_id_in_orders_to_delete})"
-    end
-
-    def truncate_order_cycle_data
-      sql_delete_from "coordinator_fees #{where_oc_id_in_ocs_to_delete}"
-      sql_delete_from "
-        exchange_variants where exchange_id
-        in (select id from exchanges #{where_oc_id_in_ocs_to_delete})"
-      sql_delete_from "
-        exchange_fees where exchange_id
-        in (select id from exchanges #{where_oc_id_in_ocs_to_delete})"
-      sql_delete_from "exchanges #{where_oc_id_in_ocs_to_delete}"
+      exit unless HighLine.new.agree(message) { |question| question.default = "N" }
     end
   end
 end

lib/tasks/data/truncate_data.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+class TruncateData
+  # This model lets us operate on the sessions DB table using ActiveRecord's
+  # methods within the scope of this service. This relies on the AR's
+  # convention where a Session model maps to a sessions table.
+  class Session < ActiveRecord::Base
+  end
+
+  def initialize(months_to_keep = nil)
+    @date = (months_to_keep || 24).to_i.months.ago
+  end
+
+  def call
+    logging do
+      truncate_inventory
+      truncate_adjustments
+      truncate_order_associations
+      truncate_order_cycle_data
+
+      sql_delete_from "spree_orders #{where_oc_id_in_ocs_to_delete}"
+
+      truncate_subscriptions
+
+      sql_delete_from "order_cycles #{where_ocs_to_delete}"
+
+      Spree::TokenizedPermission.where("created_at < '#{date}'").delete_all
+
+      remove_transient_data
+    end
+  end
+
+  private
+
+  attr_reader :date
+
+  def logging
+    Rails.logger.info("TruncateData started with truncation date #{date}")
+    yield
+    Rails.logger.info("TruncateData finished")
+  end
+
+  def truncate_order_associations
+    sql_delete_from "spree_line_items #{where_order_id_in_orders_to_delete}"
+    sql_delete_from "spree_payments #{where_order_id_in_orders_to_delete}"
+    sql_delete_from "spree_shipments #{where_order_id_in_orders_to_delete}"
+    sql_delete_from "spree_return_authorizations #{where_order_id_in_orders_to_delete}"
+  end
+
+  def remove_transient_data
+    Spree::StateChange.delete_all("created_at < '#{1.month.ago.to_date}'")
+    Spree::LogEntry.delete_all("created_at < '#{1.month.ago.to_date}'")
+    Session.delete_all("created_at < '#{2.weeks.ago.to_date}'")
+  end
+
+  def truncate_subscriptions
+    sql_delete_from "order_cycle_schedules #{where_oc_id_in_ocs_to_delete}"
+    sql_delete_from "proxy_orders #{where_oc_id_in_ocs_to_delete}"
+  end
+
+  def truncate_inventory
+    sql_delete_from "
+        spree_inventory_units #{where_order_id_in_orders_to_delete}"
+    sql_delete_from "
+        spree_inventory_units
+        where shipment_id in (select id from spree_shipments #{where_order_id_in_orders_to_delete})"
+  end
+
+  def sql_delete_from(sql)
+    ActiveRecord::Base.connection.execute("DELETE FROM #{sql}")
+  end
+
+  def where_order_id_in_orders_to_delete
+    "where order_id in (select id from spree_orders #{where_oc_id_in_ocs_to_delete})"
+  end
+
+  def where_oc_id_in_ocs_to_delete
+    "where order_cycle_id in (select id from order_cycles #{where_ocs_to_delete} )"
+  end
+
+  def where_ocs_to_delete
+    "where orders_close_at < '#{date}'"
+  end
+
+  def truncate_adjustments
+    sql_delete_from "spree_adjustments where source_type = 'Spree::Order'
+      and source_id in (select id from spree_orders #{where_oc_id_in_ocs_to_delete})"
+
+    sql_delete_from "spree_adjustments where source_type = 'Spree::Shipment'
+      and source_id in (select id from spree_shipments #{where_order_id_in_orders_to_delete})"
+
+    sql_delete_from "spree_adjustments where source_type = 'Spree::Payment'
+      and source_id in (select id from spree_payments #{where_order_id_in_orders_to_delete})"
+
+    sql_delete_from "spree_adjustments where source_type = 'Spree::LineItem'
+      and source_id in (select id from spree_line_items #{where_order_id_in_orders_to_delete})"
+  end
+
+  def truncate_order_cycle_data
+    sql_delete_from "coordinator_fees #{where_oc_id_in_ocs_to_delete}"
+    sql_delete_from "
+    exchange_variants where exchange_id
+    in (select id from exchanges #{where_oc_id_in_ocs_to_delete})"
+    sql_delete_from "
+        exchange_fees where exchange_id
+        in (select id from exchanges #{where_oc_id_in_ocs_to_delete})"
+    sql_delete_from "exchanges #{where_oc_id_in_ocs_to_delete}"
+  end
+end

spec/lib/tasks/data/truncate_data_rake_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+require 'rake'
+
+describe 'truncate_data.rake' do
+  describe ':truncate' do
+    context 'when months_to_keep is specified' do
+      it 'truncates order cycles closed earlier than months_to_keep months ago' do
+        Rake.application.rake_require 'tasks/data/truncate_data'
+        Rake::Task.define_task(:environment)
+
+        highline = instance_double(HighLine, agree: true)
+        allow(HighLine).to receive(:new).and_return(highline)
+
+        old_order_cycle = create(
+          :order_cycle,
+          orders_open_at: 7.months.ago,
+          orders_close_at: 7.months.ago + 1.day,
+        )
+        create(:order, order_cycle: old_order_cycle)
+        recent_order_cycle = create(
+          :order_cycle,
+          orders_open_at: 1.months.ago,
+          orders_close_at: 1.months.ago + 1.day,
+        )
+        create(:order, order_cycle: recent_order_cycle)
+
+        months_to_keep = 6
+        Rake.application.invoke_task "ofn:data:truncate[#{months_to_keep}]"
+
+        expect(OrderCycle.all).to contain_exactly(recent_order_cycle)
+      end
+    end
+  end
+end

spec/lib/tasks/data/truncate_data_spec.rb

@@ -0,0 +1,130 @@
+require 'spec_helper'
+require 'tasks/data/truncate_data'
+
+describe TruncateData do
+  describe '#call' do
+    before do
+      allow(Spree::ReturnAuthorization).to receive(:delete_all)
+      allow(Spree::StateChange).to receive(:delete_all)
+      allow(Spree::LogEntry).to receive(:delete_all)
+      allow(TruncateData::Session).to receive(:delete_all)
+      allow(Rails.logger).to receive(:info)
+    end
+
+    context 'when months_to_keep is not specified' do
+      it 'truncates order cycles closed earlier than 2 years ago' do
+        order_cycle = create(
+          :order_cycle, orders_open_at: 25.months.ago, orders_close_at: 25.months.ago + 1.day
+        )
+        create(:order, order_cycle: order_cycle)
+
+        TruncateData.new.call
+
+        expect(OrderCycle.all).to be_empty
+      end
+
+      it 'deletes state changes older than a month' do
+        TruncateData.new.call
+
+        expect(Spree::StateChange)
+          .to have_received(:delete_all)
+          .with("created_at < '#{1.month.ago.to_date}'")
+      end
+
+      it 'deletes log entries older than a month' do
+        TruncateData.new.call
+
+        expect(Spree::LogEntry)
+          .to have_received(:delete_all)
+          .with("created_at < '#{1.month.ago.to_date}'")
+      end
+
+      it 'deletes sessions older than two weeks' do
+        TruncateData.new.call
+
+        expect(TruncateData::Session)
+          .to have_received(:delete_all)
+          .with("created_at < '#{2.weeks.ago.to_date}'")
+      end
+    end
+
+    context 'when months_to_keep is nil' do
+      it 'truncates order cycles closed earlier than 2 years ago' do
+        order_cycle = create(
+          :order_cycle, orders_open_at: 25.months.ago, orders_close_at: 25.months.ago + 1.day
+        )
+        create(:order, order_cycle: order_cycle)
+
+        TruncateData.new(nil).call
+
+        expect(OrderCycle.all).to be_empty
+      end
+
+      it 'deletes state changes older than a month' do
+        TruncateData.new.call
+
+        expect(Spree::StateChange)
+          .to have_received(:delete_all)
+          .with("created_at < '#{1.month.ago.to_date}'")
+      end
+
+      it 'deletes log entries older than a month' do
+        TruncateData.new.call
+
+        expect(Spree::LogEntry)
+          .to have_received(:delete_all)
+          .with("created_at < '#{1.month.ago.to_date}'")
+      end
+
+      it 'deletes sessions older than two weeks' do
+        TruncateData.new.call
+
+        expect(TruncateData::Session)
+          .to have_received(:delete_all)
+          .with("created_at < '#{2.weeks.ago.to_date}'")
+      end
+    end
+
+    context 'when months_to_keep is specified' do
+      it 'truncates order cycles closed earlier than months_to_keep months ago' do
+        old_order_cycle = create(
+          :order_cycle, orders_open_at: 7.months.ago, orders_close_at: 7.months.ago + 1.day
+        )
+        create(:order, order_cycle: old_order_cycle)
+        recent_order_cycle = create(
+          :order_cycle, orders_open_at: 1.months.ago, orders_close_at: 1.months.ago + 1.day
+        )
+        create(:order, order_cycle: recent_order_cycle)
+
+        TruncateData.new(6).call
+
+        expect(OrderCycle.all).to contain_exactly(recent_order_cycle)
+      end
+
+      it 'deletes state changes older than a month' do
+        TruncateData.new.call
+
+        expect(Spree::StateChange)
+          .to have_received(:delete_all)
+          .with("created_at < '#{1.month.ago.to_date}'")
+      end
+
+      it 'deletes log entries older than a month' do
+        TruncateData.new.call
+
+        expect(Spree::LogEntry)
+          .to have_received(:delete_all)
+          .with("created_at < '#{1.month.ago.to_date}'")
+      end
+
+      it 'deletes sessions older than two weeks' do
+        TruncateData.new.call
+
+        expect(TruncateData::Session)
+          .to have_received(:delete_all)
+          .with("created_at < '#{2.weeks.ago.to_date}'")
+      end
+    end
+  end
+end
+