From f29e569f1bc4eac68524f96d9163399a1dd9bf60 Mon Sep 17 00:00:00 2001 From: Maikel Linke Date: Thu, 7 Apr 2022 14:43:54 +1000 Subject: [PATCH] Remove Paperclip migration code --- .../from_paperclip_to_active_storage.rake | 136 ------------------ ...m_paperclip_to_active_storage_rake_spec.rb | 117 --------------- 2 files changed, 253 deletions(-) delete mode 100644 lib/tasks/from_paperclip_to_active_storage.rake delete mode 100644 spec/lib/tasks/from_paperclip_to_active_storage_rake_spec.rb diff --git a/lib/tasks/from_paperclip_to_active_storage.rake b/lib/tasks/from_paperclip_to_active_storage.rake deleted file mode 100644 index edd0a2e2ee..0000000000 --- a/lib/tasks/from_paperclip_to_active_storage.rake +++ /dev/null @@ -1,136 +0,0 @@ -# frozen_string_literal: true - -namespace :from_paperclip_to_active_storage do - # This migration can't be a pure database migration because we need to know - # the location of current files which is computed by Paperclip depending on - # the `url` option. - desc "Copy data to Active Storage tables referencing Paperclip files" - task migrate: :environment do - Rails.application.eager_load! - - HasMigratingFile.migrating_models.each do |model_name| - puts "Migrating #{model_name}" - migrate_model(model_name.constantize) - end - end - - # We have a special class called ContentConfiguration which is not a model - # and therfore can't use the normal Active Storage magic. - # - # It uses `Spree::Preference`s to store all the Paperclip attributes. These - # files are stored locally and we can replace them with preferences pointing - # to an Active Storage blob. - desc "Associate ContentConfig to ActiveStorage blobs" - task copy_content_config: :environment do - [ - :logo, - :logo_mobile, - :logo_mobile_svg, - :home_hero, - :footer_logo, - ].each do |name| - migrate_content_config_file(name) - end - end - - def migrate_model(model) - duplicated_attachment_names(model).each do |name| - migrate_attachment(model, name) - end - end - - def migrate_attachment(model, name) - records_to_migrate = missing_active_storage_attachment(model, name) - - print " - #{name} (#{records_to_migrate.count}) " - - records_to_migrate.find_each do |record| - attach_paperclip(name, record) - end - - puts "" - end - - def attach_paperclip(name, record) - paperclip = record.public_send(name) - - if paperclip.respond_to?(:s3_object) - attachment = storage_record_for(name, paperclip) - record.public_send("#{name}_attachment=", attachment) - print "." - elsif File.exist?(paperclip.path) - record.attach_file(name, File.open(paperclip.path)) - record.save! - print "." - else - print "x" - end - rescue StandardError => e - puts "x" - puts e.message - end - - # Creates an Active Storage record pointing to the same file Paperclip - # stored on AWS S3. Getting the checksum requires a HEAD request. - # In my tests, I could process 100 records per minute this way. - def storage_record_for(name, paperclip) - checksum = hex_to_base64_digest(paperclip.s3_object(:original).etag.delete('"')) - - blob = ActiveStorage::Blob.new( - key: paperclip.path(:original), - filename: paperclip.original_filename, - content_type: paperclip.content_type, - metadata: {}, - byte_size: paperclip.size, - checksum: checksum, - created_at: paperclip.updated_at, - ) - ActiveStorage::Attachment.new( - name: name, - blob: blob, - created_at: paperclip.updated_at, - ) - end - - def migrate_content_config_file(name) - paperclip = ContentConfig.public_send(name) - - return if ContentConfig.public_send("#{name}_blob") - return if paperclip.path.blank? || !paperclip.exists? - - blob = ActiveStorage::Blob.create_and_upload!( - io: File.open(paperclip.path), - filename: paperclip.original_filename, - content_type: paperclip.content_type, - identify: false, - ) - - ContentConfig.public_send("#{name}_blob_id=", blob.id) - puts "Copied #{name}" - end - - def duplicated_attachment_names(model) - paperclip_attachments = model.attachment_definitions.keys.map(&:to_s) - active_storage_attachments = model.attachment_reflections.keys - - only_paperclip = paperclip_attachments - active_storage_attachments - only_active_storage = active_storage_attachments - paperclip_attachments - both = paperclip_attachments & active_storage_attachments - - puts "WARNING: not migrating #{only_paperclip}" if only_paperclip.present? - puts "WARNING: no source for #{only_active_storage}" if only_active_storage.present? - - both - end - - # Records with Paperclip but without an Active storage attachment yet - def missing_active_storage_attachment(model, attachment) - model.where.not("#{attachment}_file_name" => [nil, ""]). - left_outer_joins("#{attachment}_attachment".to_sym). - where(active_storage_attachments: { id: nil }) - end - - def hex_to_base64_digest(hexdigest) - [[hexdigest].pack("H*")].pack("m0") - end -end diff --git a/spec/lib/tasks/from_paperclip_to_active_storage_rake_spec.rb b/spec/lib/tasks/from_paperclip_to_active_storage_rake_spec.rb deleted file mode 100644 index 210607034a..0000000000 --- a/spec/lib/tasks/from_paperclip_to_active_storage_rake_spec.rb +++ /dev/null @@ -1,117 +0,0 @@ -# frozen_string_literal: true - -require "spec_helper" -require "rake" - -describe "from_paperclip_to_active_storage.rake" do - include FileHelper - - let(:file) { Rack::Test::UploadedFile.new(black_logo_file, 'image/png') } - let(:s3_config) { - { - url: ":s3_alias_url", - storage: :s3, - s3_credentials: { - access_key_id: "A...A", - secret_access_key: "H...H", - }, - s3_headers: { "Cache-Control" => "max-age=31557600" }, - bucket: "ofn", - s3_protocol: "https", - s3_host_alias: "ofn.s3.us-east-1.amazonaws.com", - - # This is for easier testing: - path: "/:id/:style/:basename.:extension", - } - } - - before(:all) do - Rake.application.rake_require "tasks/from_paperclip_to_active_storage" - Rake::Task.define_task(:environment) - end - - describe ":migrate" do - it "creates Active Storage records for existing images on disk" do - image = Spree::Image.create!(attachment: file) - image.attachment_attachment.delete - image.attachment_blob.delete - - expect { - run_task "from_paperclip_to_active_storage:migrate" - }.to change { - image.reload.active_storage_attachment.attached? - }.to(true) - end - - it "creates Active Storage records for images on AWS S3" do - attachment_definition = Spree::Image.attachment_definitions[:attachment] - allow(Spree::Image).to receive(:attachment_definitions).and_return( - attachment: attachment_definition.merge(s3_config) - ) - allow(Rails.application.config.active_storage). - to receive(:service).and_return(:test_amazon) - - stub_request(:put, /amazonaws/).to_return(status: 200, body: "", headers: {}) - stub_request(:head, /amazonaws/).to_return( - status: 200, body: "", - headers: { - "ETag" => '"87b0a401e077485a078c0a15ceb7eb39"' - } - ) - stub_request(:put, /amazonaws/).to_return(status: 200, body: "", headers: {}) - - image = Spree::Image.create!(attachment: file) - image.attachment_attachment.delete - image.attachment_blob.delete - - expect { - run_task "from_paperclip_to_active_storage:migrate" - }.to change { - image.reload.active_storage_attachment.attached? - }.to(true) - - # The checksum can be computed with Active Storage: - # - # ActiveStorage::Blob.build_after_unfurling( - # io: file, identify: false, - # filename: "logo-black.png", - # content_type: "image/png", - # ).checksum - expect(image.attachment_blob.checksum).to eq "h7CkAeB3SFoHjAoVzrfrOQ==" - end - end - - describe ":copy_content_config" do - it "doesn't copy default images" do - run_task "from_paperclip_to_active_storage:copy_content_config" - - expect(ContentConfig.logo_blob).to eq nil - end - - it "copies uploaded images" do - ContentConfig.logo = file - ContentConfig.logo.save - - run_task "from_paperclip_to_active_storage:copy_content_config" - - expect(ContentConfig.logo_blob).to be_a ActiveStorage::Blob - end - - it "doesn't copy twice" do - ContentConfig.logo = file - ContentConfig.logo.save - - expect { - run_task "from_paperclip_to_active_storage:copy_content_config" - run_task "from_paperclip_to_active_storage:copy_content_config" - }.to change { - ActiveStorage::Blob.count - }.by(1) - end - end - - def run_task(name) - Rake::Task[name].reenable - Rake.application.invoke_task(name) - end -end