Merge pull request #13772 from mkllnk/dfc-webhook

Import farm data from LiteFarm
This commit is contained in:
Maikel
2025-12-12 16:59:30 +11:00
committed by GitHub
22 changed files with 891 additions and 91 deletions

View File

@@ -15,8 +15,8 @@ module Admin
def index
# Fetch DFC catalog JSON for preview
@catalog_url = params.require(:catalog_url).strip
@catalog_json = api.call(@catalog_url)
catalog = DfcCatalog.from_json(@catalog_json)
@catalog_data = api.call(@catalog_url)
catalog = DfcCatalog.from_json(@catalog_data)
# Render table and let user decide which ones to import.
@items = list_products(catalog)

View File

@@ -4,7 +4,7 @@ module ManagerInvitations
extend ActiveSupport::Concern
def create_new_manager(email, enterprise)
password = Devise.friendly_token
password = SecureRandom.base58(64)
new_user = Spree::User.create(email:, unconfirmed_email: email, password:)
new_user.reset_password_token = Devise.friendly_token
# Same time as used in Devise's lib/devise/models/recoverable.rb.

View File

@@ -77,6 +77,7 @@ class Enterprise < ApplicationRecord
has_many :connected_apps, dependent: :destroy
has_many :dfc_permissions, dependent: :destroy
has_one :custom_tab, dependent: :destroy
has_one :semantic_link, as: :subject, dependent: :delete
delegate :latitude, :longitude, :city, :state_name, to: :address

View File

@@ -12,7 +12,7 @@
= form_with url: main_app.import_admin_dfc_product_imports_path, html: { "data-controller": "checked" } do |form|
-# This is a very inefficient way of holding a json blob. Maybe base64 encode or store as a temporary file
= form.hidden_field :enterprise_id, value: @enterprise.id
= form.hidden_field :catalog_json, value: @catalog_json
= form.hidden_field :catalog_json, value: @catalog_data.to_json
%table
%thead

View File

@@ -0,0 +1,63 @@
# frozen_string_literal: true
# Webhook events
module DfcProvider
class EventsController < DfcProvider::ApplicationController
rescue_from JSON::ParserError, with: -> do
head :bad_request
end
# Trigger a webhook event.
#
# The only supported event is a `refresh` event of permissions.
# It means that our permissions to access data on another platform changed.
# We will need to pull the updated data.
def create
unless current_user.is_a? ApiUser
unauthorized "You need to authenticate as authorised platform (client_id)."
return
end
unless current_user.id == "lf-dev"
unauthorized "Your client_id is not authorised on this platform."
return
end
event = JSON.parse(request.body.read)
enterprises_url = event["enterpriseUrlid"]
if enterprises_url.blank?
render status: :bad_request, json: {
success: false,
message: "Missing parameter `enterpriseUrlid`",
}
return
end
importer = DfcImporter.new
importer.import_enterprise_profiles(current_user.id, enterprises_url)
if importer.errors.blank?
render json: { success: true }
else
render json: { success: true, messages: error_messages(importer.errors) }
end
end
private
def unauthorized(message)
render_message(:unauthorized, message)
end
def render_message(status, message)
render status:, json: { success: false, message: }
end
def error_messages(errors)
errors.map do |error|
id = error.record.try(:semantic_link)&.semantic_id
"#{id}: #{error.message}"
end
end
end
end

View File

@@ -0,0 +1,39 @@
# frozen_string_literal: true
# Fetch data from another platform and store it locally.
class DfcImporter
attr_reader :errors
def import_enterprise_profiles(platform, enterprises_url)
raise "unsupported platform" if platform != "lf-dev"
api = DfcPlatformRequest.new(platform)
body = api.call(enterprises_url)
graph = DfcIo.import(body).to_a
farms = graph.select { |item| item.semanticType == "dfc-b:Enterprise" }
farms.each { |farm| import_profile(farm) }
end
def import_profile(farm)
owner = find_or_import_user(farm)
enterprise = EnterpriseImporter.new(owner, farm).import
enterprise.save! if enterprise.changed?
enterprise.address.save! if enterprise.address.changed?
rescue ActiveRecord::RecordInvalid => e
Alert.raise(e, farm: DfcIo.export(farm))
@errors ||= []
@errors << e
end
def find_or_import_user(farm)
email = farm.mainContact.emails.first
user = Spree::User.find_by(email:)
return user if user
Spree::User.create!(
email:,
password: SecureRandom.base58(64),
)
end
end

View File

@@ -0,0 +1,32 @@
# frozen_string_literal: true
# Request a JSON document from a DFC API authenticating as platform.
class DfcPlatformRequest
def initialize(platform)
@platform = platform
end
def call(url, data = nil, method: nil)
OidcRequest.new(request_token).call(url, data, method:).body
end
def request_token
connection = Faraday.new(
request: { timeout: 5 },
) do |f|
f.request :url_encoded
f.response :json
f.response :raise_error
end
url = ApiUser.token_endpoint(@platform)
data = {
grant_type: "client_credentials",
client_id: ENV.fetch("OPENID_APP_ID", nil),
client_secret: ENV.fetch("OPENID_APP_SECRET", nil),
scope: "ReadEnterprise",
}
response = connection.post(url, data)
response.body["access_token"]
end
end

View File

@@ -1,8 +1,5 @@
# frozen_string_literal: true
require "private_address_check"
require "private_address_check/tcpsocket_ext"
# Request a JSON document from a DFC API with authentication.
#
# All DFC API interactions are authenticated via OIDC tokens. If the user's
@@ -15,16 +12,15 @@ class DfcRequest
def call(url, data = nil, method: nil)
begin
response = request(url, data, method:)
request = OidcRequest.new(@user.oidc_account.token)
response = request.call(url, data, method:)
rescue Faraday::UnauthorizedError, Faraday::ForbiddenError
raise unless token_stale?
# If access was denied and our token is stale then refresh and retry:
refresh_access_token!
response = request(url, data, method:)
rescue Faraday::ServerError => e
Alert.raise(e, { dfc_request: { data: } })
raise
request = OidcRequest.new(@user.oidc_account.token)
response = request.call(url, data, method:)
end
response.body
@@ -32,41 +28,10 @@ class DfcRequest
private
def request(url, data = nil, method: nil)
only_public_connections do
if method == :put
connection.put(url, data)
elsif data
connection.post(url, data)
else
connection.get(url)
end
end
end
def token_stale?
@user.oidc_account.updated_at < 15.minutes.ago
end
def connection
Faraday.new(
request: { timeout: 30 },
headers: {
'Content-Type' => 'application/json',
'Authorization' => "Bearer #{@user.oidc_account.token}",
}
) do |f|
# Configure Faraday to raise errors on status 4xx and 5xx responses.
f.response :raise_error
end
end
def only_public_connections(&)
return yield if Rails.env.development?
PrivateAddressCheck.only_public_connections(&)
end
def refresh_access_token!
strategy = OmniAuth::Strategies::OpenIDConnect.new(
Rails.application,

View File

@@ -0,0 +1,98 @@
# frozen_string_literal: true
require "private_address_check"
require "private_address_check/tcpsocket_ext"
class EnterpriseImporter
def initialize(owner, dfc_enterprise)
@owner = owner
@dfc_enterprise = dfc_enterprise
end
def import
enterprise = find || new
apply(enterprise)
enterprise
end
def find
semantic_id = @dfc_enterprise.semanticId
@owner.owned_enterprises.includes(:semantic_link)
.find_by(semantic_link: { semantic_id: })
end
def new
@owner.owned_enterprises.new(
address: Spree::Address.new,
semantic_link: SemanticLink.new(semantic_id: @dfc_enterprise.semanticId),
is_primary_producer: true,
visible: "public",
)
end
def apply(enterprise)
address = @dfc_enterprise.localizations.first
country = find_country(address)
enterprise.name = @dfc_enterprise.name
enterprise.address.assign_attributes(
address1: address.street,
city: address.city,
zipcode: address.postalCode,
state: find_state(country, address),
country:,
)
enterprise.email_address = @dfc_enterprise.emails.first
enterprise.description = @dfc_enterprise.description
enterprise.phone = @dfc_enterprise.phoneNumbers.first&.phoneNumber
enterprise.website = @dfc_enterprise.websites.first
apply_social_media(enterprise)
apply_logo(enterprise)
end
def apply_social_media(enterprise)
attributes = {}
@dfc_enterprise.socialMedias.each do |media|
attributes[media.name.downcase] = media.url
end
attributes["twitter"] = attributes.delete("x") if attributes.key?("x")
enterprise_attributes = attributes.slice(*SocialMediaBuilder::NAMES)
enterprise.assign_attributes(enterprise_attributes)
end
def apply_logo(enterprise)
link = @dfc_enterprise.logo
logo = enterprise.logo
return if link.blank?
return if logo.blob && (logo.blob.custom_metadata&.fetch("origin", nil) == link)
url = URI.parse(link)
filename = File.basename(url.path)
metadata = { custom: { origin: link } }
PrivateAddressCheck.only_public_connections do
logo.attach(io: url.open, filename:, metadata:)
end
rescue StandardError
# Any URL parsing or network error shouldn't impact the import
# at all. Maybe we'll add UX for error handling later.
nil
end
def find_country(address)
country = address.country
country = country[:path] if country.is_a?(Hash)
Spree::Country.find_by(iso3: country.to_s[-3..]) ||
Spree::Country.find_by(name: country) ||
Spree::Country.first
end
def find_state(country, address)
country.states.find_by(name: address.region) || country.states.first
end
end

View File

@@ -0,0 +1,53 @@
# frozen_string_literal: true
require "private_address_check"
require "private_address_check/tcpsocket_ext"
# Request a JSON document with an OIDC token.
class OidcRequest
def initialize(token)
@token = token
end
def call(url, data = nil, method: nil)
request(url, data, method:)
rescue StandardError => e
Alert.raise(e, { dfc_request: { data: } })
raise
end
private
def request(url, data = nil, method: nil)
only_public_connections do
if method == :put
connection.put(url, data)
elsif data
connection.post(url, data)
else
connection.get(url)
end
end
end
def connection
Faraday.new(
request: { timeout: 30 },
headers: {
'Authorization' => "Bearer #{@token}",
}
) do |f|
f.request :json
f.response :json
# Configure Faraday to raise errors on status 4xx and 5xx responses.
f.response :raise_error
end
end
def only_public_connections(&)
return yield if Rails.env.development?
PrivateAddressCheck.only_public_connections(&)
end
end

View File

@@ -1,54 +1,15 @@
# frozen_string_literal: true
require "private_address_check"
require "private_address_check/tcpsocket_ext"
# Call a webhook to notify a data proxy about changes in our data.
class ProxyNotifier
def refresh(platform, enterprise_url)
PrivateAddressCheck.only_public_connections do
notify_proxy(platform, enterprise_url)
end
end
def request_token(platform)
connection = Faraday.new(
request: { timeout: 5 },
) do |f|
f.request :url_encoded
f.response :json
f.response :raise_error
end
url = ApiUser.token_endpoint(platform)
data = {
grant_type: "client_credentials",
client_id: ENV.fetch("OPENID_APP_ID", nil),
client_secret: ENV.fetch("OPENID_APP_SECRET", nil),
scope: "ReadEnterprise",
}
response = connection.post(url, data)
response.body["access_token"]
end
def notify_proxy(platform, enterprise_url)
token = request_token(platform)
endpoint = ApiUser.webhook_url(platform)
data = {
eventType: "refresh",
enterpriseUrlid: enterprise_url,
scope: "ReadEnterprise",
}
connection = Faraday.new(
request: { timeout: 10 },
headers: {
'Authorization' => "Bearer #{token}",
}
) do |f|
f.request :json
f.response :json
f.response :raise_error
end
connection.post(ApiUser.webhook_url(platform), data)
api = DfcPlatformRequest.new(platform)
api.call(endpoint, data)
end
end

View File

@@ -12,6 +12,7 @@ DfcProvider::Engine.routes.draw do
resources :enterprise_groups, only: [:index, :show] do
resources :affiliated_by, only: [:create, :destroy], module: 'enterprise_groups'
end
resources :events, only: [:create]
resources :persons, only: [:show]
resources :supplied_products, only: [:index]
resources :product_groups, only: [:show]

View File

@@ -0,0 +1,127 @@
# frozen_string_literal: true
require_relative "../swagger_helper"
RSpec.describe "Events", swagger_doc: "dfc.yaml" do
include_context "authenticated as platform" do
let(:access_token) {
file_fixture("fdc_access_token.jwt").read
}
end
path "/api/dfc/events" do
post "Create Event" do
consumes "application/json"
produces "application/json"
parameter name: :event, in: :body, schema: {
example: {
eventType: "refresh",
enterpriseUrlid: "https://api.beta.litefarm.org/dfc/enterprises/",
scope: "ReadEnterprise",
}
}
response "400", "bad request" do
describe "with missing request body" do
around do |example|
# Rswag expects all required parameters to be supplied with `let`
# but we want to send a request without the request body parameter.
parameters = example.metadata[:operation][:parameters]
example.metadata[:operation][:parameters] = []
example.run
example.metadata[:operation][:parameters] = parameters
end
run_test!
end
describe "with empty request body" do
let(:event) { nil }
run_test!
end
describe "with missing parameter" do
let(:event) { { eventType: "refresh" } }
run_test!
end
end
response "401", "unauthorised" do
describe "as normal user" do
let(:Authorization) { nil }
let(:event) { { eventType: "refresh" } }
before { login_as create(:oidc_user) }
run_test!
end
describe "as other platform" do
let(:access_token) {
file_fixture("startinblox_access_token.jwt").read
}
let(:event) { { eventType: "refresh" } }
before { login_as create(:oidc_user) }
run_test!
end
end
response "200", "success" do
let(:event) do |example|
example.metadata[:operation][:parameters].first[:schema][:example]
end
before do
stub_request(:post, %r{openid-connect/token$})
end
describe "when some records fail" do
before do
body = {
'@context': "https://www.datafoodconsortium.org",
'@graph': [
{
'@id': "http://some-id",
'@type': "dfc-b:Enterprise",
'dfc-b:hasMainContact': "http://some-person",
'dfc-b:hasAddress': "http://address",
},
{
'@id': "http://some-person",
'@type': "dfc-b:Person",
'dfc-b:email': "community@litefarm.org",
},
{
'@id': "http://address",
'@type': "dfc-b:Address",
},
]
}.to_json
stub_request(:get, "https://api.beta.litefarm.org/dfc/enterprises/")
.to_return(body:)
end
run_test! do
expect(json_response["success"]).to eq true
expect(json_response["messages"].first)
.to match "http://some-id: Validation failed: Address address1 can't be blank"
end
end
describe "importing an empty list" do
before do
stub_request(:get, "https://api.beta.litefarm.org/dfc/enterprises/")
.to_return(body: "[]")
end
run_test! do
expect(json_response["success"]).to eq true
end
end
end
end
end
end

View File

@@ -0,0 +1,45 @@
# frozen_string_literal: true
require_relative "../spec_helper"
# These tests depend on valid OpenID Connect client credentials in your
# `.env.test.local` file.
#
# OPENID_APP_ID="..."
# OPENID_APP_SECRET="..."
RSpec.describe DfcImporter do
let(:endpoint) { "https://api.beta.litefarm.org/dfc/enterprises/" }
let(:semantic_id) {
"https://api.beta.litefarm.org/dfc/enterprises/23bfd9b1-98b5-4b91-88e5-efa7cb36219d"
}
it "fetches a list of enterprises", :vcr do
expect {
subject.import_enterprise_profiles("lf-dev", endpoint)
}.to have_enqueued_mail(Spree::UserMailer, :confirmation_instructions).exactly(7)
.and have_enqueued_mail(EnterpriseMailer, :welcome).exactly(6)
# You can show the emails in your browser.
# Consider creating a test helper if you find this useful elsewhere.
# allow(ApplicationMailer).to receive(:delivery_method).and_return(:letter_opener)
# perform_enqueued_jobs(only: ActionMailer::MailDeliveryJob)
# Repeating works without creating duplicates:
expect {
subject.import_enterprise_profiles("lf-dev", endpoint)
}.not_to have_enqueued_mail
enterprise = Enterprise.joins(:semantic_link).find_by(semantic_link: { semantic_id: })
expect(enterprise.name).to eq "DFC Test Farm Beta (All Supplied Fields)"
expect(enterprise.email_address).to eq "dfcshop@example.com"
expect(enterprise.logo.blob.content_type).to eq "image/webp"
expect(enterprise.logo.blob.byte_size).to eq 8974
expect(enterprise.visible).to eq "public"
expect(subject.errors.count).to eq 2
expect(subject.errors.first.record.semantic_link.semantic_id)
.to eq "https://api.beta.litefarm.org/dfc/enterprises/13152ea2-8d19-4309-a443-c95d8879d299"
expect(subject.errors.first.message)
.to eq "Validation failed: Address zipcode can't be blank, Address is invalid"
end
end

View File

@@ -0,0 +1,19 @@
# frozen_string_literal: true
require_relative "../spec_helper"
# These tests depend on valid OpenID Connect client credentials in your
# `.env.test.local` file if you want to update the VCR cassettes.
#
# OPENID_APP_ID="..."
# OPENID_APP_SECRET="..."
RSpec.describe DfcPlatformRequest do
subject { DfcPlatformRequest.new(platform) }
let(:platform) { "cqcm-dev" }
it "receives an access token", :vcr do
token = subject.request_token
expect(token).to be_a String
expect(token.length).to be > 20
end
end

View File

@@ -0,0 +1,63 @@
# frozen_string_literal: true
require_relative "../spec_helper"
RSpec.describe EnterpriseImporter do
subject { EnterpriseImporter.new(owner, dfc_enterprise) }
let(:owner) { Spree::User.new }
let(:dfc_enterprise) {
DataFoodConsortium::Connector::Enterprise.new(
"litefarm.org",
name: "Test Farm",
localizations: [
DataFoodConsortium::Connector::Address.new(
nil,
region: "Victoria",
country: {
scheme: "http",
host: "publications.europa.eu",
path: "/resource/authority/country/AUS",
}
)
],
socialMedias: [
DataFoodConsortium::Connector::SocialMedia.new(
nil,
name: "Facebook",
url: "dfc_test_farm",
)
],
)
}
it "assigns data to a new enterprise object" do
enterprise = subject.import
expect(enterprise.id).to eq nil
expect(enterprise.semantic_link.semantic_id).to eq "litefarm.org"
expect(enterprise.name).to eq "Test Farm"
expect(enterprise.address.state.name).to eq "Victoria"
expect(enterprise.address.country.name).to eq "Australia"
expect(enterprise.facebook).to eq "dfc_test_farm"
end
it "understands old country names" do
dfc_enterprise.localizations[0].country = "France"
dfc_enterprise.localizations[0].region = "Aquitaine"
enterprise = subject.import
expect(enterprise.id).to eq nil
expect(enterprise.address.country.name).to eq "France"
expect(enterprise.address.state.name).to eq "Aquitaine"
end
it "ignores errors during image import" do
dfc_enterprise.logo = "invalid url"
enterprise = subject.import
expect(enterprise.name).to eq "Test Farm"
expect(enterprise.logo.attached?).to eq false
end
end

View File

@@ -0,0 +1,32 @@
# frozen_string_literal: true
require_relative "../spec_helper"
RSpec.describe OidcRequest do
subject(:api) { OidcRequest.new("some-token") }
it "gets a DFC document" do
stub_request(:get, "http://example.net/api").
to_return(status: 200, body: '{"@context":"/"}')
expect(api.call("http://example.net/api").body).to eq '{"@context":"/"}'
end
it "posts a DFC document" do
json = '{"name":"new season apples"}'
stub_request(:post, "http://example.net/api").
with(body: json).
to_return(status: 201) # Created
expect(api.call("http://example.net/api", json).body).to eq ""
end
it "reports and raises server errors" do
stub_request(:get, "http://example.net/api").to_return(status: 500)
expect(Bugsnag).to receive(:notify)
expect { api.call("http://example.net/api") }
.to raise_error(Faraday::ServerError)
end
end

View File

@@ -11,12 +11,6 @@ RSpec.describe ProxyNotifier do
let(:platform) { "cqcm-dev" }
let(:enterprise_url) { "http://ofn.example.net/api/dfc/enterprises/10000" }
it "receives an access token", :vcr do
token = subject.request_token(platform)
expect(token).to be_a String
expect(token.length).to be > 20
end
it "notifies the proxy", :vcr do
# The test server is not reachable by the notified server.
# If you don't have valid credentials, you'll get an unauthorized error.

File diff suppressed because one or more lines are too long

View File

@@ -7,6 +7,15 @@ VCR.configure do |config|
config.hook_into :webmock
config.configure_rspec_metadata!
# Change recording mode during development:
#
# VCR_RECORD=new_episodes ./bin/rspec spec/example_spec.rb
# VCR_RECORD=all ./bin/rspec spec/example_spec.rb
#
if ENV.fetch("VCR_RECORD", nil)
config.default_cassette_options = { record: ENV.fetch("VCR_RECORD").to_sym }
end
# Chrome calls a lot of services and they trip us up.
config.ignore_hosts(
"localhost", "127.0.0.1", "0.0.0.0",

View File

@@ -579,6 +579,47 @@ paths:
dfc-b:URL: https://facebook.com/user
'404':
description: not found
"/api/dfc/events":
post:
summary: Create Event
parameters: []
tags:
- Events
responses:
'400':
description: bad request
content:
application/json:
examples:
test_example:
value:
success: false
message: Missing parameter `enterpriseUrlid`
'401':
description: unauthorised
content:
application/json:
examples:
test_example:
value:
success: false
message: Your client_id is not authorised on this platform.
'200':
description: success
content:
application/json:
examples:
test_example:
value:
success: true
requestBody:
content:
application/json:
schema:
example:
eventType: refresh
enterpriseUrlid: https://api.beta.litefarm.org/dfc/enterprises/
scope: ReadEnterprise
"/api/dfc/enterprises/{enterprise_id}/offers/{id}":
parameters:
- name: enterprise_id