object: factor out fetching functions into Pleroma.Object.Fetcher module

This commit is contained in:
William Pitcock 2018-12-01 22:53:10 +00:00
parent e8caecb5c7
commit f85949cc69
7 changed files with 81 additions and 134 deletions

View file

@ -0,0 +1,69 @@
defmodule Pleroma.Object.Fetcher do
alias Pleroma.{Object, Repo}
alias Pleroma.Object.Containment
alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.OStatus
require Logger
@httpoison Application.get_env(:pleroma, :httpoison)
# TODO:
# This will create a Create activity, which we need internally at the moment.
def fetch_object_from_id(id) do
if object = Object.get_cached_by_ap_id(id) do
{:ok, object}
else
Logger.info("Fetching #{id} via AP")
with {:ok, data} <- fetch_and_contain_remote_object_from_id(id),
nil <- Object.normalize(data),
params <- %{
"type" => "Create",
"to" => data["to"],
"cc" => data["cc"],
"actor" => data["actor"] || data["attributedTo"],
"object" => data
},
:ok <- Containment.contain_origin(id, params),
{:ok, activity} <- Transmogrifier.handle_incoming(params) do
{:ok, Object.normalize(activity.data["object"])}
else
{:error, {:reject, nil}} ->
{:reject, nil}
object = %Object{} ->
{:ok, object}
_e ->
Logger.info("Couldn't get object via AP, trying out OStatus fetching...")
case OStatus.fetch_activity_from_url(id) do
{:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"])}
e -> e
end
end
end
end
def fetch_and_contain_remote_object_from_id(id) do
Logger.info("Fetching #{id} via AP")
with true <- String.starts_with?(id, "http"),
{:ok, %{body: body, status_code: code}} when code in 200..299 <-
@httpoison.get(
id,
[Accept: "application/activity+json"],
follow_redirect: true,
timeout: 10000,
recv_timeout: 20000
),
{:ok, data} <- Jason.decode(body),
:ok <- Containment.contain_origin_from_id(id, data) do
{:ok, data}
else
e ->
{:error, e}
end
end
end

View file

@ -1,6 +1,6 @@
defmodule Pleroma.Web.ActivityPub.ActivityPub do defmodule Pleroma.Web.ActivityPub.ActivityPub do
alias Pleroma.{Activity, Repo, Object, Upload, User, Notification} alias Pleroma.{Activity, Repo, Object, Upload, User, Notification}
alias Pleroma.Object.Containment alias Pleroma.Object.Fetcher
alias Pleroma.Web.ActivityPub.{Transmogrifier, MRF} alias Pleroma.Web.ActivityPub.{Transmogrifier, MRF}
alias Pleroma.Web.WebFinger alias Pleroma.Web.WebFinger
alias Pleroma.Web.Federator alias Pleroma.Web.Federator
@ -629,7 +629,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end end
def fetch_and_prepare_user_from_ap_id(ap_id) do def fetch_and_prepare_user_from_ap_id(ap_id) do
with {:ok, data} <- fetch_and_contain_remote_object_from_id(ap_id) do with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do
user_data_from_user_object(data) user_data_from_user_object(data)
else else
e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}") e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
@ -723,65 +723,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
) )
end end
# TODO:
# This will create a Create activity, which we need internally at the moment.
def fetch_object_from_id(id) do
if object = Object.get_cached_by_ap_id(id) do
{:ok, object}
else
Logger.info("Fetching #{id} via AP")
with {:ok, data} <- fetch_and_contain_remote_object_from_id(id),
nil <- Object.normalize(data),
params <- %{
"type" => "Create",
"to" => data["to"],
"cc" => data["cc"],
"actor" => data["actor"] || data["attributedTo"],
"object" => data
},
:ok <- Containment.contain_origin(id, params),
{:ok, activity} <- Transmogrifier.handle_incoming(params) do
{:ok, Object.normalize(activity.data["object"])}
else
{:error, {:reject, nil}} ->
{:reject, nil}
object = %Object{} ->
{:ok, object}
_e ->
Logger.info("Couldn't get object via AP, trying out OStatus fetching...")
case OStatus.fetch_activity_from_url(id) do
{:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"])}
e -> e
end
end
end
end
def fetch_and_contain_remote_object_from_id(id) do
Logger.info("Fetching #{id} via AP")
with true <- String.starts_with?(id, "http"),
{:ok, %{body: body, status_code: code}} when code in 200..299 <-
@httpoison.get(
id,
[Accept: "application/activity+json"],
follow_redirect: true,
timeout: 10000,
recv_timeout: 20000
),
{:ok, data} <- Jason.decode(body),
:ok <- Containment.contain_origin_from_id(id, data) do
{:ok, data}
else
e ->
{:error, e}
end
end
def is_public?(activity) do def is_public?(activity) do
"https://www.w3.org/ns/activitystreams#Public" in (activity.data["to"] ++ "https://www.w3.org/ns/activitystreams#Public" in (activity.data["to"] ++
(activity.data["cc"] || [])) (activity.data["cc"] || []))

View file

@ -1,6 +1,7 @@
defmodule Pleroma.Web.ActivityPub.ActivityPubController do defmodule Pleroma.Web.ActivityPub.ActivityPubController do
use Pleroma.Web, :controller use Pleroma.Web, :controller
alias Pleroma.{User, Object} alias Pleroma.{User, Object}
alias Pleroma.Object.Fetcher
alias Pleroma.Web.ActivityPub.{ObjectView, UserView} alias Pleroma.Web.ActivityPub.{ObjectView, UserView}
alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Relay alias Pleroma.Web.ActivityPub.Relay
@ -122,7 +123,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
"Signature missing or not from author, relayed Create message, fetching object from source" "Signature missing or not from author, relayed Create message, fetching object from source"
) )
ActivityPub.fetch_object_from_id(params["object"]["id"]) Fetcher.fetch_object_from_id(params["object"]["id"])
json(conn, "ok") json(conn, "ok")
end end

View file

@ -4,7 +4,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
""" """
alias Pleroma.User alias Pleroma.User
alias Pleroma.Object alias Pleroma.Object
alias Pleroma.Object.Containment alias Pleroma.Object.{Containment, Fetcher}
alias Pleroma.Activity alias Pleroma.Activity
alias Pleroma.Repo alias Pleroma.Repo
alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.ActivityPub
@ -529,8 +529,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
def handle_incoming(_), do: :error def handle_incoming(_), do: :error
def fetch_obj_helper(id) when is_bitstring(id), do: ActivityPub.fetch_object_from_id(id) def fetch_obj_helper(id) when is_bitstring(id), do: Fetcher.fetch_object_from_id(id)
def fetch_obj_helper(obj) when is_map(obj), do: ActivityPub.fetch_object_from_id(obj["id"]) def fetch_obj_helper(obj) when is_map(obj), do: Fetcher.fetch_object_from_id(obj["id"])
def get_obj_helper(id) do def get_obj_helper(id) do
if object = Object.normalize(id), do: {:ok, object}, else: nil if object = Object.normalize(id), do: {:ok, object}, else: nil

View file

@ -1,6 +1,7 @@
defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
use Pleroma.Web, :controller use Pleroma.Web, :controller
alias Pleroma.{Repo, Object, Activity, User, Notification, Stats} alias Pleroma.{Repo, Object, Activity, User, Notification, Stats}
alias Pleroma.Object.Fetcher
alias Pleroma.Web alias Pleroma.Web
alias Pleroma.Web.MastodonAPI.{StatusView, AccountView, MastodonView, ListView, FilterView} alias Pleroma.Web.MastodonAPI.{StatusView, AccountView, MastodonView, ListView, FilterView}
alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.ActivityPub
@ -658,7 +659,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
def status_search(query) do def status_search(query) do
fetched = fetched =
if Regex.match?(~r/https?:/, query) do if Regex.match?(~r/https?:/, query) do
with {:ok, object} <- ActivityPub.fetch_object_from_id(query) do with {:ok, object} <- Fetcher.fetch_object_from_id(query) do
[Activity.get_create_activity_by_object_ap_id(object.data["id"])] [Activity.get_create_activity_by_object_ap_id(object.data["id"])]
else else
_e -> [] _e -> []
@ -986,7 +987,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
def login(conn, _) do def login(conn, _) do
with {:ok, app} <- get_or_make_app() do with {:ok, app} <- get_or_make_app() do
path = path =
o_auth_path(conn, :authorize, o_auth_path(
conn,
:authorize,
response_type: "code", response_type: "code",
client_id: app.client_id, client_id: app.client_id,
redirect_uri: ".", redirect_uri: ".",

View file

@ -372,43 +372,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
end end
end end
describe "fetching an object" do
test "it fetches an object" do
{:ok, object} =
ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
assert activity = Activity.get_create_activity_by_object_ap_id(object.data["id"])
assert activity.data["id"]
{:ok, object_again} =
ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
assert [attachment] = object.data["attachment"]
assert is_list(attachment["url"])
assert object == object_again
end
test "it works with objects only available via Ostatus" do
{:ok, object} = ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873")
assert activity = Activity.get_create_activity_by_object_ap_id(object.data["id"])
assert activity.data["id"]
{:ok, object_again} =
ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873")
assert object == object_again
end
test "it correctly stitches up conversations between ostatus and ap" do
last = "https://mstdn.io/users/mayuutann/statuses/99568293732299394"
{:ok, object} = ActivityPub.fetch_object_from_id(last)
object = Object.get_by_ap_id(object.data["inReplyTo"])
assert object
end
end
describe "following / unfollowing" do describe "following / unfollowing" do
test "creates a follow activity" do test "creates a follow activity" do
follower = insert(:user) follower = insert(:user)
@ -530,15 +493,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
end end
end end
test "it can fetch plume articles" do
{:ok, object} =
ActivityPub.fetch_object_from_id(
"https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"
)
assert object
end
describe "update" do describe "update" do
test "it creates an update activity with the new user data" do test "it creates an update activity with the new user data" do
user = insert(:user) user = insert(:user)
@ -560,15 +514,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
end end
end end
test "it can fetch peertube videos" do
{:ok, object} =
ActivityPub.fetch_object_from_id(
"https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
)
assert object
end
def data_uri do def data_uri do
File.read!("test/fixtures/avatar_data_uri") File.read!("test/fixtures/avatar_data_uri")
end end

View file

@ -894,10 +894,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
end end
describe "actor origin containment" do describe "actor origin containment" do
test "it rejects objects with a bogus origin" do
{:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity.json")
end
test "it rejects activities which reference objects with bogus origins" do test "it rejects activities which reference objects with bogus origins" do
data = %{ data = %{
"@context" => "https://www.w3.org/ns/activitystreams", "@context" => "https://www.w3.org/ns/activitystreams",
@ -911,10 +907,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
:error = Transmogrifier.handle_incoming(data) :error = Transmogrifier.handle_incoming(data)
end end
test "it rejects objects when attributedTo is wrong (variant 1)" do
{:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity2.json")
end
test "it rejects activities which reference objects that have an incorrect attribution (variant 1)" do test "it rejects activities which reference objects that have an incorrect attribution (variant 1)" do
data = %{ data = %{
"@context" => "https://www.w3.org/ns/activitystreams", "@context" => "https://www.w3.org/ns/activitystreams",
@ -928,10 +920,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
:error = Transmogrifier.handle_incoming(data) :error = Transmogrifier.handle_incoming(data)
end end
test "it rejects objects when attributedTo is wrong (variant 2)" do
{:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity3.json")
end
test "it rejects activities which reference objects that have an incorrect attribution (variant 2)" do test "it rejects activities which reference objects that have an incorrect attribution (variant 2)" do
data = %{ data = %{
"@context" => "https://www.w3.org/ns/activitystreams", "@context" => "https://www.w3.org/ns/activitystreams",