Merge pull request #12274 from mkllnk/dfc-product-import

Import products from DFC catalog

.env.test

@@ -14,3 +14,4 @@ SITE_URL="test.host"
 
 OPENID_APP_ID="test-provider"
 OPENID_APP_SECRET="12345"
+OPENID_REFRESH_TOKEN="dummy-refresh-token"

app/controllers/admin/dfc_product_imports_controller.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "private_address_check"
+require "private_address_check/tcpsocket_ext"
+
+module Admin
+  class DfcProductImportsController < Spree::Admin::BaseController
+    # Define model class for `can?` permissions:
+    def model_class
+      self.class
+    end
+
+    def index
+      # The plan:
+      #
+      # * Fetch DFC catalog as JSON from URL.
+      enterprise = OpenFoodNetwork::Permissions.new(spree_current_user)
+        .managed_product_enterprises.is_primary_producer
+        .find(params.require(:enterprise_id))
+
+      catalog_url = params.require(:catalog_url)
+
+      json_catalog = DfcRequest.new(spree_current_user).get(catalog_url)
+      graph = DfcIo.import(json_catalog)
+
+      # * First step: import all products for given enterprise.
+      # * Second step: render table and let user decide which ones to import.
+      imported = graph.map do |subject|
+        import_product(subject, enterprise)
+      end
+
+      @count = imported.compact.count
+    end
+
+    private
+
+    # Most of this code is the same as in the DfcProvider::SuppliedProductsController.
+    def import_product(subject, enterprise)
+      return unless subject.is_a? DataFoodConsortium::Connector::SuppliedProduct
+
+      variant = SuppliedProductBuilder.import_variant(subject, enterprise)
+      product = variant.product
+
+      product.save! if product.new_record?
+      variant.save! if variant.new_record?
+
+      variant
+    end
+  end
+end

app/controllers/admin/product_import_controller.rb

@@ -10,6 +10,8 @@ module Admin
       @product_categories = Spree::Taxon.order('name ASC').pluck(:name).uniq
       @tax_categories = Spree::TaxCategory.order('name ASC').pluck(:name)
       @shipping_categories = Spree::ShippingCategory.order('name ASC').pluck(:name)
+      @producers = OpenFoodNetwork::Permissions.new(spree_current_user).
+        managed_product_enterprises.is_primary_producer.by_name.to_a
     end
 
     def import

app/models/semantic_link.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# Link a Spree::Variant to an external DFC SuppliedProduct.
+class SemanticLink < ApplicationRecord
+  belongs_to :variant, class_name: "Spree::Variant"
+
+  validates :semantic_id, presence: true
+end

app/models/spree/ability.rb

@@ -239,6 +239,8 @@ module Spree
       can [:admin, :index, :guide, :import, :save, :save_data,
            :validate_data, :reset_absent_products], ProductImport::ProductImporter
 
+      can [:admin, :index], ::Admin::DfcProductImportsController
+
       # Reports page
       can [:admin, :index, :show], ::Admin::ReportsController
       can [:admin, :show, :customers, :orders_and_distributors, :group_buys, :payments,

app/models/spree/variant.rb

@@ -56,6 +56,7 @@ module Spree
     has_many :exchanges, through: :exchange_variants
     has_many :variant_overrides, dependent: :destroy
     has_many :inventory_items, dependent: :destroy
+    has_many :semantic_links, dependent: :delete_all
 
     localize_number :price, :weight
 

app/views/admin/dfc_product_imports/index.html.haml

@@ -0,0 +1,7 @@
+- content_for :page_title do
+  #{t(".title")}
+
+= render partial: 'spree/admin/shared/product_sub_menu'
+
+%p= t(".imported_products")
+= @count

app/views/admin/product_import/_dfc_import_form.html.haml

@@ -0,0 +1,16 @@
+%h3= t(".title")
+%br
+
+= form_with url: main_app.admin_dfc_product_imports_path, method: :get do |form|
+  = form.label :enterprise_id, t(".enterprise")
+  %span.required *
+  %br
+  = form.select :enterprise_id, options_from_collection_for_select(@producers, :id, :name, @producers.first&.id), { "data-controller": "tom-select", class: "primary" }
+  %br
+  %br
+  = form.label :catalog_url, t(".catalog_url")
+  %br
+  = form.text_field :catalog_url, size: 60
+  %br
+  %br
+  = form.submit t(".import")

app/views/admin/product_import/index.html.haml

@@ -13,3 +13,5 @@
 %br
 
 = render 'upload_form'
+
+= render 'dfc_import_form' if spree_current_user.oidc_account.present?

config/locales/en.yml

@@ -757,6 +757,10 @@ en:
         user_guide: User Guide
         map: Map
 
+    dfc_product_imports:
+      index:
+        title: "Importing a DFC product catalog"
+        imported_products: "Imported products:"
     enterprise_fees:
       index:
         title: "Enterprise Fees"
@@ -929,6 +933,11 @@ en:
         product_categories: Product Categories
         tax_categories: Tax Categories
         shipping_categories: Shipping Categories
+      dfc_import_form:
+        title: "Import from DFC catalog"
+        enterprise: "Enterprise"
+        catalog_url: "DFC catalog URL"
+        import: "Import"
       import:
         review: Review
         import: Import

config/routes/admin.rb

@@ -67,6 +67,8 @@ Openfoodnetwork::Application.routes.draw do
     post '/product_import/save_data', to: 'product_import#save_data', as: 'product_import_save_async'
     post '/product_import/reset_absent', to: 'product_import#reset_absent_products', as: 'product_import_reset_async'
 
+    resources :dfc_product_imports, only: [:index]
+
     constraints FeatureToggleConstraint.new(:admin_style_v3) do
       resources :products, to: 'products_v3#index', only: :index do
         patch :bulk_update, on: :collection

db/migrate/20240105043228_create_semantic_links.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class CreateSemanticLinks < ActiveRecord::Migration[7.0]
+  def change
+    create_table :semantic_links do |t|
+      t.references :variant, null: false, foreign_key: { to_table: :spree_variants }
+      t.string :semantic_id, null: false
+
+      t.timestamps
+    end
+  end
+end

db/schema.rb

@@ -400,6 +400,14 @@ ActiveRecord::Schema[7.0].define(version: 2024_02_13_044159) do
     t.datetime "updated_at", precision: nil, null: false
   end
 
+  create_table "semantic_links", force: :cascade do |t|
+    t.bigint "variant_id", null: false
+    t.string "semantic_id", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["variant_id"], name: "index_semantic_links_on_variant_id"
+  end
+
   create_table "sessions", id: :serial, force: :cascade do |t|
     t.string "session_id", limit: 255, null: false
     t.text "data"
@@ -1168,6 +1176,7 @@ ActiveRecord::Schema[7.0].define(version: 2024_02_13_044159) do
   add_foreign_key "proxy_orders", "spree_orders", column: "order_id", name: "order_id_fk"
   add_foreign_key "proxy_orders", "subscriptions", name: "proxy_orders_subscription_id_fk"
   add_foreign_key "report_rendering_options", "spree_users", column: "user_id"
+  add_foreign_key "semantic_links", "spree_variants", column: "variant_id"
   add_foreign_key "spree_addresses", "spree_countries", column: "country_id", name: "spree_addresses_country_id_fk"
   add_foreign_key "spree_addresses", "spree_states", column: "state_id", name: "spree_addresses_state_id_fk"
   add_foreign_key "spree_inventory_units", "spree_orders", column: "order_id", name: "spree_inventory_units_order_id_fk", on_delete: :cascade

engines/dfc_provider/app/services/dfc_request.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "private_address_check"
+require "private_address_check/tcpsocket_ext"
+
+# Request a JSON document from a DFC API with authentication.
+#
+# All DFC API interactions are authenticated via OIDC tokens. If the user's
+# access token is expired, we try to get a new one with the user's refresh
+# token.
+class DfcRequest
+  def initialize(user)
+    @user = user
+  end
+
+  def get(url)
+    response = request(url)
+
+    return response.body if response.status == 200
+
+    return "" if @user.oidc_account.updated_at > 15.minutes.ago
+
+    refresh_access_token!
+
+    response = request(url)
+    response.body
+  end
+
+  private
+
+  def request(url)
+    connection = Faraday.new(
+      request: { timeout: 30 },
+      headers: {
+        'Content-Type' => 'application/json',
+        'Authorization' => "Bearer #{@user.oidc_account.token}",
+      }
+    )
+
+    only_public_connections do
+      connection.get(url)
+    end
+  end
+
+  def only_public_connections(&)
+    return yield if Rails.env.development?
+
+    PrivateAddressCheck.only_public_connections(&)
+  end
+
+  def refresh_access_token!
+    strategy = OmniAuth::Strategies::OpenIDConnect.new(
+      Rails.application,
+      Devise.omniauth_configs[:openid_connect].options
+      # Don't try to call `Devise.omniauth(:openid_connect)` first.
+      # It results in an empty config hash and we lose our config.
+    )
+    client = strategy.client
+    client.token_endpoint = strategy.config.token_endpoint
+    client.refresh_token = @user.oidc_account.refresh_token
+
+    token = client.access_token!
+
+    @user.oidc_account.update!(
+      token: token.access_token,
+      refresh_token: token.refresh_token
+    )
+  end
+end

engines/dfc_provider/app/services/supplied_product_builder.rb

@@ -38,6 +38,9 @@ class SuppliedProductBuilder < DfcBuilder
       product.supplier = supplier
       product.ensure_standard_variant
       product.variants.first
+    end.tap do |variant|
+      link = supplied_product.semanticId
+      variant.semantic_links.new(semantic_id: link) if link.present?
     end
   end
 
@@ -87,8 +90,11 @@ class SuppliedProductBuilder < DfcBuilder
   end
 
   def self.taxon(supplied_product)
-    dfc_id = supplied_product.productType.semanticId
-    Spree::Taxon.find_by(dfc_id: )
+    dfc_id = supplied_product.productType&.semanticId
+
+    # Every product needs a primary taxon to be valid. So if we don't have
+    # one or can't find it we just take a random one.
+    Spree::Taxon.find_by(dfc_id:) || Spree::Taxon.first
   end
 
   private_class_method :product_type, :taxon

engines/dfc_provider/spec/services/dfc_request_spec.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require_relative "../spec_helper"
+
+describe DfcRequest do
+  subject(:api) { DfcRequest.new(user) }
+
+  let(:user) { build(:oidc_user) }
+  let(:account) { user.oidc_account }
+
+  it "gets a DFC document" do
+    stub_request(:get, "http://example.net/api").
+      to_return(status: 200, body: '{"@context":"/"}')
+
+    expect(api.get("http://example.net/api")).to eq '{"@context":"/"}'
+  end
+
+  it "refreshes the access token on fail", vcr: true do
+    # Live VCR recordings require the following secret ENV variables:
+    # - OPENID_APP_ID
+    # - OPENID_APP_SECRET
+    # - OPENID_REFRESH_TOKEN
+    # You can set them in the .env.test.local file.
+
+    stub_request(:get, "http://example.net/api").
+      to_return(status: 401)
+
+    # A refresh is only attempted if the token is stale.
+    account.refresh_token = ENV.fetch("OPENID_REFRESH_TOKEN")
+    account.updated_at = 1.day.ago
+
+    expect {
+      api.get("http://example.net/api")
+    }.to change {
+      account.token
+    }.and change {
+      account.refresh_token
+    }
+  end
+
+  it "doesn't try to refresh the token when it's still fresh" do
+    stub_request(:get, "http://example.net/api").
+      to_return(status: 401)
+
+    user.oidc_account.updated_at = 1.minute.ago
+
+    expect(api.get("http://example.net/api")).to eq ""
+
+    # Trying to reach the OIDC server via network request to refresh the token
+    # would raise errors because we didn't setup Webmock or VCR.
+    # The absence of errors makes this test pass.
+  end
+end

engines/dfc_provider/spec/services/supplied_product_builder_spec.rb

@@ -64,14 +64,6 @@ describe SuppliedProductBuilder do
 
         expect(product.productType).to eq soft_drink
       end
-
-      context "when no taxon set" do
-        let(:taxon) { nil }
-
-        it "returns nil" do
-          expect(product.productType).to be_nil
-        end
-      end
     end
 
     it "assigns an image_url type" do
@@ -131,16 +123,6 @@ describe SuppliedProductBuilder do
 
         expect(product.primary_taxon).to eq(taxon)
       end
-
-      describe "when no matching taxon" do
-        let(:product_type) { DfcLoader.connector.PRODUCT_TYPES.DRINK }
-
-        it "set the taxon to nil" do
-          product = builder.import_product(supplied_product)
-
-          expect(product.primary_taxon).to be_nil
-        end
-      end
     end
   end
 
@@ -161,11 +143,19 @@ describe SuppliedProductBuilder do
     let(:product_type) { DfcLoader.connector.PRODUCT_TYPES.VEGETABLE.NON_LOCAL_VEGETABLE }
 
     it "creates a new Spree::Product and variant" do
+      create(:taxon)
+
       expect(imported_variant).to be_a(Spree::Variant)
+      expect(imported_variant).to be_valid
       expect(imported_variant.id).to be_nil
+      expect(imported_variant.semantic_links.size).to eq 1
+
+      link = imported_variant.semantic_links[0]
+      expect(link.semantic_id).to eq "https://example.net/tomato"
 
       imported_product = imported_variant.product
       expect(imported_product).to be_a(Spree::Product)
+      expect(imported_product).to be_valid
       expect(imported_product.id).to be_nil
       expect(imported_product.name).to eq("Tomato")
       expect(imported_product.description).to eq("Awesome tomato")

spec/models/semantic_link_spec.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe SemanticLink, type: :model do
+  it { is_expected.to belong_to :variant }
+  it { is_expected.to validate_presence_of(:semantic_id) }
+end

spec/models/spree/variant_spec.rb

@@ -6,6 +6,8 @@ require 'spree/localized_number'
 describe Spree::Variant do
   subject(:variant) { build(:variant) }
 
+  it { is_expected.to have_many :semantic_links }
+
   context "validations" do
     it "should validate price is greater than 0" do
       variant.price = -1

spec/support/vcr_setup.rb

@@ -16,6 +16,9 @@ VCR.configure do |config|
     STRIPE_ACCOUNT
     STRIPE_CLIENT_ID
     STRIPE_ENDPOINT_SECRET
+    OPENID_APP_ID
+    OPENID_APP_SECRET
+    OPENID_REFRESH_TOKEN
   ].each do |env_var|
     config.filter_sensitive_data("<HIDDEN-#{env_var}>") { ENV.fetch(env_var, nil) }
   end
@@ -25,4 +28,16 @@ VCR.configure do |config|
   config.filter_sensitive_data('<HIDDEN-CLIENT-SECRET>') { |interaction|
     interaction.response.body.match(/"client_secret": "(pi_.+)"/)&.public_send(:[], 1)
   }
+  config.filter_sensitive_data('<HIDDEN-AUTHORIZATION-HEADER>') { |interaction|
+    interaction.request.headers['Authorization']&.public_send(:[], 0)
+  }
+  config.filter_sensitive_data('<HIDDEN-OPENID-TOKEN>') { |interaction|
+    interaction.response.body.match(/"access_token":"([^"]+)"/)&.public_send(:[], 1)
+  }
+  config.filter_sensitive_data('<HIDDEN-OPENID-TOKEN>') { |interaction|
+    interaction.response.body.match(/"id_token":"([^"]+)"/)&.public_send(:[], 1)
+  }
+  config.filter_sensitive_data('<HIDDEN-OPENID-TOKEN>') { |interaction|
+    interaction.response.body.match(/"refresh_token":"([^"]+)"/)&.public_send(:[], 1)
+  }
 end

spec/system/admin/dfc_product_import_spec.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: false
+
+require 'system_helper'
+require_relative '../../../engines/dfc_provider/spec/support/authorization_helper'
+
+describe "DFC Product Import" do
+  include AuthorizationHelper
+
+  let(:user) { create(:oidc_user, owned_enterprises: [enterprise]) }
+  let(:enterprise) { create(:supplier_enterprise) }
+  let(:source_product) { create(:product, supplier: enterprise) }
+
+  before do
+    login_as user
+    source_product # to be imported
+    allow(PrivateAddressCheck).to receive(:private_address?).and_return(false)
+    user.oidc_account.update!(token: allow_token_for(email: user.email))
+  end
+
+  it "imports from given catalog" do
+    visit admin_product_import_path
+
+    select enterprise.name, from: "Enterprise"
+
+    # We are testing against our own catalog for now but we want to replace
+    # this with the URL of another app when available.
+    host = Rails.application.default_url_options[:host]
+    url = "http://#{host}/api/dfc/enterprises/#{enterprise.id}/catalog_items"
+    fill_in "catalog_url", with: url
+
+    # By feeding our own catalog to the import, we are effectively cloning the
+    # products. But the DFC product references the spree_product_id which
+    # make the importer create a variant for that product instead of creating
+    # a new independent product.
+    expect {
+      click_button "Import"
+    }.to change {
+      source_product.variants.count
+    }.by(1)
+
+    expect(page).to have_content "Importing a DFC product catalog"
+    expect(page).to have_content "Imported products: 1"
+  end
+end