Add sanitize and truncate tasks to support loading live data into other environments safely

db/sanitize_data.rb

@@ -1,43 +0,0 @@
-
-def update_address(address, user)
-  unless address.nil?
-    address.firstname = user[:first_name]
-    address.lastname = user[:last_name]
-    address.phone = user[:phone]
-    address.save!
-  end
-end
-
-def sanitize_data
-  canned_users = [ { :first_name => "Bob", :last_name => "jones", :email => "bob@jones.com", :phone => "0123456789" },
-   { :first_name => "cindy", :last_name => "rest", :email => "cindy@gmail.com", :phone => "0123456789" },
-   { :first_name => "Pete", :last_name => "smith", :email => "pete@gmail.com", :phone => "0123456789" },
-   { :first_name => "Tony", :last_name => "ballantyne", :email => "tony@gmail.com", :phone => "0123456789" },
-   { :first_name => "Ben", :last_name => "raven", :email => "ben@gmail.com", :phone => "0123456789" },
-   { :first_name => "Robyn", :last_name => "monster", :email => "robyn@gmail.com", :phone => "0123456789" },
-   { :first_name => "Nako", :last_name => "tolkein", :email => "nako@gmail.com", :phone => "0123456789" },
-   { :first_name => "Helen", :last_name => "mitcham", :email => "helen@gmail.com", :phone => "0123456789" },
-   { :first_name => "Emma", :last_name => "low", :email => "emma@gmail.com", :phone => "0123456789" },
-   { :first_name => "Mandy", :last_name => "Trust", :email => "Mandy@trust.com", :phone => "0123456789" } ]
-
-   Spree::Order.all.each_with_index do |order, index|
-    canned_user = canned_users[index%canned_users.size]
-    puts "updating order #{order.id} with #{canned_user[:first_name]}"
-
-    order.email = canned_user[:email]
-
-    update_address(order.bill_address, canned_user)
-    update_address(order.ship_address, canned_user)
-    order.save!
-  end
-
-  Spree::User.all.each_with_index do |user, index|
-    unless user.email == "admin@openfoodweb.org"
-     canned_user = canned_users[index%canned_users.size]
-     puts "updating user #{user.id} with #{canned_user[:first_name]}"
-
-     user.email = "#{canned_user[:email]}#{index}"
-     user.save!
-   end
- end
-end

lib/tasks/data/sanitize_data.rake

@@ -0,0 +1,56 @@
+require 'highline'
+
+namespace :ofn do
+  namespace :data do
+    desc 'Sanitize data'
+    task sanitize: :environment do
+      guard_and_warn
+
+      Spree::User.update_all("email = concat(id, '_ofn_user@example.com'),
+                              login = concat(id, '_ofn_user@example.com'),
+                              unconfirmed_email = concat(id, '_ofn_user@example.com')")
+      Spree::Customer.update_all("email = concat(id, '_ofn_customer@example.com'),
+                                  name = concat('Customer Number ', id)")
+      Spree::Order.update_all("email = concat(id, '_ofn_order@example.com')")
+      Spree::Address.update_all("
+        firstname = concat('Ms. Number', id), lastname = 'Jones',  phone = '01234567890',
+        alternative_phone = '01234567890', address1 = 'Dummy address',
+        address2 = 'Dummy address continuation', city = 'Dummy City', zipcode = '0000',
+        company = null, latitude = null, longitude = null")
+      Spree::TokenizedPermission.update_all("token = null")
+
+      # Sanitize payments related entities
+      Spree::PaymentMethod.update_all("name = concat('Dummy Payment Method', id),
+                                       description = name")
+      Spree::CreditCard.update_all("
+        month = 12, year = 2020, start_month = 12, start_year = 2000,
+        cc_type = 'VISA', first_name = 'Dummy', last_name = 'Dummy', last_digits = '2543'")
+      Spree::Payment.update_all("response_code = null, avs_response = null,
+                                 cvv_response_code = null, identifier = null,
+                                 cvv_response_message = null")
+      Spree::PaypalExpressCheckout.update_all("token = null")
+      StripeAccount.delete_all
+      ActiveRecord::Base.connection.execute("delete from spree_paypal_accounts")
+
+      # Update environment in mail methods and payment methods
+      ActiveRecord::Base.connection.execute("update spree_mail_methods set environment = '#{Rails.env}'")
+      Spree::PaymentMethod.update_all("environment = '#{Rails.env}'")
+
+      # Delete all preferences that may contain sensitive information
+      Spree::Preference
+        .where("key like '%gateway%' OR key like '%billing_integration%' OR key like '%s3%'")
+        .delete_all
+    end
+
+    def guard_and_warn
+      if Rails.env.production?
+        Rails.logger.info("This task cannot be executed in production")
+        exit
+      end
+
+      message = "\n <%= color('This will permanently change DB contents', :yellow) %>,
+                are you sure you want to proceed? (y/N)"
+      exit unless HighLine.new.agree(message) { |q| q.default = "n" }
+    end
+  end
+end

lib/tasks/data/truncate_data.rake

@@ -0,0 +1,75 @@
+# This task can be used to significantly reduce the size of a database
+#   This is used for example when loading live data into a staging server
+#   This way the staging server is not overloaded with too much data
+namespace :ofn do
+  namespace :data do
+    desc 'Truncate data'
+    task truncate: :environment do
+      guard_and_warn
+
+      date = 3.months.ago
+
+      where_ocs_to_delete = "where orders_close_at < '#{date}'"
+      where_oc_id_in_ocs_to_delete = "
+        where order_cycle_id in (select id from order_cycles #{where_ocs_to_delete} )"
+      where_order_id_in_orders_to_delete = "
+        where order_id in (select id from spree_orders #{where_oc_id_in_ocs_to_delete})"
+
+      sql_delete_from "
+        spree_inventory_units #{where_order_id_in_orders_to_delete}"
+      sql_delete_from "
+        spree_adjustments where source_type = 'Spree::Order'
+        and source_id in (select id from spree_orders #{where_oc_id_in_ocs_to_delete})"
+      sql_delete_from "
+        spree_adjustments where source_type = 'Spree::Shipment'
+        and source_id in (select id from spree_shipments #{where_order_id_in_orders_to_delete})"
+      sql_delete_from "
+        spree_adjustments where source_type = 'Spree::Payment'
+        and source_id in (select id from spree_payments #{where_order_id_in_orders_to_delete})"
+      sql_delete_from "
+        spree_adjustments where source_type = 'Spree::LineItem'
+        and source_id in (select id from spree_line_items #{where_order_id_in_orders_to_delete})"
+      sql_delete_from "spree_line_items #{where_order_id_in_orders_to_delete}"
+      sql_delete_from "spree_payments #{where_order_id_in_orders_to_delete}"
+      sql_delete_from "spree_shipments #{where_order_id_in_orders_to_delete}"
+      sql_delete_from "billable_periods"
+      sql_delete_from "account_invoices"
+      Spree::ReturnAuthorization.delete_all
+
+      sql_delete_from "coordinator_fees #{where_oc_id_in_ocs_to_delete}"
+      sql_delete_from "
+        exchange_variants where exchange_id
+        in (select id from exchanges #{where_oc_id_in_ocs_to_delete})"
+      sql_delete_from "
+        exchange_fees where exchange_id
+        in (select id from exchanges #{where_oc_id_in_ocs_to_delete})"
+      sql_delete_from "exchanges #{where_oc_id_in_ocs_to_delete}"
+      sql_delete_from "proxy_orders #{where_oc_id_in_ocs_to_delete}"
+
+      sql_delete_from "spree_orders #{where_oc_id_in_ocs_to_delete}"
+      sql_delete_from "order_cycle_schedules #{where_oc_id_in_ocs_to_delete}"
+      sql_delete_from "order_cycles #{where_ocs_to_delete}"
+
+      # Truncating addresses like this takes many hours to run on top of 300k records
+      #   The sanitize task will work well with the ful range of 300k addresses
+      # sql_delete_from "spree_addresses where
+      #  id not in (select address_id from enterprise_groups) and
+      #  id not in (select address_id from enterprises) and
+      #  id not in (select bill_address_id from spree_orders where bill_address_id is not null) and
+      #  id not in (select ship_address_id from spree_orders where ship_address_id is not null) and
+      #  id not in (select address_id from spree_shipments where address_id is not null) and
+      #  id not in (select address_id from spree_shipments where address_id is not null) and
+      #  id not in (select bill_address_id from customers where bill_address_id is not null) and
+      #  id not in (select ship_address_id from customers where ship_address_id is not null)"
+
+      Spree::TokenizedPermission.where("created_at < '#{date}'").delete_all
+      Spree::StateChange.delete_all
+      Spree::LogEntry.delete_all
+      sql_delete_from "sessions"
+    end
+
+    def sql_delete_from(sql)
+      ActiveRecord::Base.connection.execute("delete from #{sql}")
+    end
+  end
+end