From f85949cc696685496a8e4e17aebeb81168ede41a Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 1 Dec 2018 22:53:10 +0000 Subject: [PATCH] object: factor out fetching functions into Pleroma.Object.Fetcher module --- lib/pleroma/object/fetcher.ex | 69 +++++++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 63 +---------------- .../activity_pub/activity_pub_controller.ex | 3 +- .../web/activity_pub/transmogrifier.ex | 6 +- .../mastodon_api/mastodon_api_controller.ex | 7 +- test/web/activity_pub/activity_pub_test.exs | 55 --------------- test/web/activity_pub/transmogrifier_test.exs | 12 ---- 7 files changed, 81 insertions(+), 134 deletions(-) create mode 100644 lib/pleroma/object/fetcher.ex diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex new file mode 100644 index 000000000..c27cb1577 --- /dev/null +++ b/lib/pleroma/object/fetcher.ex @@ -0,0 +1,69 @@ +defmodule Pleroma.Object.Fetcher do + alias Pleroma.{Object, Repo} + alias Pleroma.Object.Containment + alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.OStatus + + require Logger + + @httpoison Application.get_env(:pleroma, :httpoison) + + # TODO: + # This will create a Create activity, which we need internally at the moment. + def fetch_object_from_id(id) do + if object = Object.get_cached_by_ap_id(id) do + {:ok, object} + else + Logger.info("Fetching #{id} via AP") + + with {:ok, data} <- fetch_and_contain_remote_object_from_id(id), + nil <- Object.normalize(data), + params <- %{ + "type" => "Create", + "to" => data["to"], + "cc" => data["cc"], + "actor" => data["actor"] || data["attributedTo"], + "object" => data + }, + :ok <- Containment.contain_origin(id, params), + {:ok, activity} <- Transmogrifier.handle_incoming(params) do + {:ok, Object.normalize(activity.data["object"])} + else + {:error, {:reject, nil}} -> + {:reject, nil} + + object = %Object{} -> + {:ok, object} + + _e -> + Logger.info("Couldn't get object via AP, trying out OStatus fetching...") + + case OStatus.fetch_activity_from_url(id) do + {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"])} + e -> e + end + end + end + end + + def fetch_and_contain_remote_object_from_id(id) do + Logger.info("Fetching #{id} via AP") + + with true <- String.starts_with?(id, "http"), + {:ok, %{body: body, status_code: code}} when code in 200..299 <- + @httpoison.get( + id, + [Accept: "application/activity+json"], + follow_redirect: true, + timeout: 10000, + recv_timeout: 20000 + ), + {:ok, data} <- Jason.decode(body), + :ok <- Containment.contain_origin_from_id(id, data) do + {:ok, data} + else + e -> + {:error, e} + end + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 517cf4b46..fefefc320 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1,6 +1,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.{Activity, Repo, Object, Upload, User, Notification} - alias Pleroma.Object.Containment + alias Pleroma.Object.Fetcher alias Pleroma.Web.ActivityPub.{Transmogrifier, MRF} alias Pleroma.Web.WebFinger alias Pleroma.Web.Federator @@ -629,7 +629,7 @@ def user_data_from_user_object(data) do end def fetch_and_prepare_user_from_ap_id(ap_id) do - with {:ok, data} <- fetch_and_contain_remote_object_from_id(ap_id) do + with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do user_data_from_user_object(data) else e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}") @@ -723,65 +723,6 @@ def publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do ) end - # TODO: - # This will create a Create activity, which we need internally at the moment. - def fetch_object_from_id(id) do - if object = Object.get_cached_by_ap_id(id) do - {:ok, object} - else - Logger.info("Fetching #{id} via AP") - - with {:ok, data} <- fetch_and_contain_remote_object_from_id(id), - nil <- Object.normalize(data), - params <- %{ - "type" => "Create", - "to" => data["to"], - "cc" => data["cc"], - "actor" => data["actor"] || data["attributedTo"], - "object" => data - }, - :ok <- Containment.contain_origin(id, params), - {:ok, activity} <- Transmogrifier.handle_incoming(params) do - {:ok, Object.normalize(activity.data["object"])} - else - {:error, {:reject, nil}} -> - {:reject, nil} - - object = %Object{} -> - {:ok, object} - - _e -> - Logger.info("Couldn't get object via AP, trying out OStatus fetching...") - - case OStatus.fetch_activity_from_url(id) do - {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"])} - e -> e - end - end - end - end - - def fetch_and_contain_remote_object_from_id(id) do - Logger.info("Fetching #{id} via AP") - - with true <- String.starts_with?(id, "http"), - {:ok, %{body: body, status_code: code}} when code in 200..299 <- - @httpoison.get( - id, - [Accept: "application/activity+json"], - follow_redirect: true, - timeout: 10000, - recv_timeout: 20000 - ), - {:ok, data} <- Jason.decode(body), - :ok <- Containment.contain_origin_from_id(id, data) do - {:ok, data} - else - e -> - {:error, e} - end - end - def is_public?(activity) do "https://www.w3.org/ns/activitystreams#Public" in (activity.data["to"] ++ (activity.data["cc"] || [])) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 3570a75cb..7b7c0e090 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -1,6 +1,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do use Pleroma.Web, :controller alias Pleroma.{User, Object} + alias Pleroma.Object.Fetcher alias Pleroma.Web.ActivityPub.{ObjectView, UserView} alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Relay @@ -122,7 +123,7 @@ def inbox(conn, %{"type" => "Create"} = params) do "Signature missing or not from author, relayed Create message, fetching object from source" ) - ActivityPub.fetch_object_from_id(params["object"]["id"]) + Fetcher.fetch_object_from_id(params["object"]["id"]) json(conn, "ok") end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 1b5e57294..e76e29b95 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do """ alias Pleroma.User alias Pleroma.Object - alias Pleroma.Object.Containment + alias Pleroma.Object.{Containment, Fetcher} alias Pleroma.Activity alias Pleroma.Repo alias Pleroma.Web.ActivityPub.ActivityPub @@ -529,8 +529,8 @@ def handle_incoming( def handle_incoming(_), do: :error - def fetch_obj_helper(id) when is_bitstring(id), do: ActivityPub.fetch_object_from_id(id) - def fetch_obj_helper(obj) when is_map(obj), do: ActivityPub.fetch_object_from_id(obj["id"]) + def fetch_obj_helper(id) when is_bitstring(id), do: Fetcher.fetch_object_from_id(id) + def fetch_obj_helper(obj) when is_map(obj), do: Fetcher.fetch_object_from_id(obj["id"]) def get_obj_helper(id) do if object = Object.normalize(id), do: {:ok, object}, else: nil diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 90225460b..71390be0d 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1,6 +1,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do use Pleroma.Web, :controller alias Pleroma.{Repo, Object, Activity, User, Notification, Stats} + alias Pleroma.Object.Fetcher alias Pleroma.Web alias Pleroma.Web.MastodonAPI.{StatusView, AccountView, MastodonView, ListView, FilterView} alias Pleroma.Web.ActivityPub.ActivityPub @@ -658,7 +659,7 @@ def unblock_domain(%{assigns: %{user: blocker}} = conn, %{"domain" => domain}) d def status_search(query) do fetched = if Regex.match?(~r/https?:/, query) do - with {:ok, object} <- ActivityPub.fetch_object_from_id(query) do + with {:ok, object} <- Fetcher.fetch_object_from_id(query) do [Activity.get_create_activity_by_object_ap_id(object.data["id"])] else _e -> [] @@ -986,7 +987,9 @@ def login(conn, %{"code" => code}) do def login(conn, _) do with {:ok, app} <- get_or_make_app() do path = - o_auth_path(conn, :authorize, + o_auth_path( + conn, + :authorize, response_type: "code", client_id: app.client_id, redirect_uri: ".", diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 231a334f9..bc9fcc75d 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -372,43 +372,6 @@ test "fetches the latest Follow activity" do end end - describe "fetching an object" do - test "it fetches an object" do - {:ok, object} = - ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") - - assert activity = Activity.get_create_activity_by_object_ap_id(object.data["id"]) - assert activity.data["id"] - - {:ok, object_again} = - ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") - - assert [attachment] = object.data["attachment"] - assert is_list(attachment["url"]) - - assert object == object_again - end - - test "it works with objects only available via Ostatus" do - {:ok, object} = ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873") - assert activity = Activity.get_create_activity_by_object_ap_id(object.data["id"]) - assert activity.data["id"] - - {:ok, object_again} = - ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873") - - assert object == object_again - end - - test "it correctly stitches up conversations between ostatus and ap" do - last = "https://mstdn.io/users/mayuutann/statuses/99568293732299394" - {:ok, object} = ActivityPub.fetch_object_from_id(last) - - object = Object.get_by_ap_id(object.data["inReplyTo"]) - assert object - end - end - describe "following / unfollowing" do test "creates a follow activity" do follower = insert(:user) @@ -530,15 +493,6 @@ test "it filters broken threads" do end end - test "it can fetch plume articles" do - {:ok, object} = - ActivityPub.fetch_object_from_id( - "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/" - ) - - assert object - end - describe "update" do test "it creates an update activity with the new user data" do user = insert(:user) @@ -560,15 +514,6 @@ test "it creates an update activity with the new user data" do end end - test "it can fetch peertube videos" do - {:ok, object} = - ActivityPub.fetch_object_from_id( - "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" - ) - - assert object - end - def data_uri do File.read!("test/fixtures/avatar_data_uri") end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index a71a2c5b1..ea9d9fe58 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -894,10 +894,6 @@ test "it fixes the actor URL property to be a proper URI" do end describe "actor origin containment" do - test "it rejects objects with a bogus origin" do - {:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity.json") - end - test "it rejects activities which reference objects with bogus origins" do data = %{ "@context" => "https://www.w3.org/ns/activitystreams", @@ -911,10 +907,6 @@ test "it rejects activities which reference objects with bogus origins" do :error = Transmogrifier.handle_incoming(data) end - test "it rejects objects when attributedTo is wrong (variant 1)" do - {:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity2.json") - end - test "it rejects activities which reference objects that have an incorrect attribution (variant 1)" do data = %{ "@context" => "https://www.w3.org/ns/activitystreams", @@ -928,10 +920,6 @@ test "it rejects activities which reference objects that have an incorrect attri :error = Transmogrifier.handle_incoming(data) end - test "it rejects objects when attributedTo is wrong (variant 2)" do - {:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity3.json") - end - test "it rejects activities which reference objects that have an incorrect attribution (variant 2)" do data = %{ "@context" => "https://www.w3.org/ns/activitystreams",