object: factor out fetching functions into Pleroma.Object.Fetcher module

lib/pleroma/object/fetcher.ex

@@ -0,0 +1,69 @@
+defmodule Pleroma.Object.Fetcher do
+  alias Pleroma.{Object, Repo}
+  alias Pleroma.Object.Containment
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+  alias Pleroma.Web.OStatus
+
+  require Logger
+
+  @httpoison Application.get_env(:pleroma, :httpoison)
+
+  # TODO:
+  # This will create a Create activity, which we need internally at the moment.
+  def fetch_object_from_id(id) do
+    if object = Object.get_cached_by_ap_id(id) do
+      {:ok, object}
+    else
+      Logger.info("Fetching #{id} via AP")
+
+      with {:ok, data} <- fetch_and_contain_remote_object_from_id(id),
+           nil <- Object.normalize(data),
+           params <- %{
+             "type" => "Create",
+             "to" => data["to"],
+             "cc" => data["cc"],
+             "actor" => data["actor"] || data["attributedTo"],
+             "object" => data
+           },
+           :ok <- Containment.contain_origin(id, params),
+           {:ok, activity} <- Transmogrifier.handle_incoming(params) do
+        {:ok, Object.normalize(activity.data["object"])}
+      else
+        {:error, {:reject, nil}} ->
+          {:reject, nil}
+
+        object = %Object{} ->
+          {:ok, object}
+
+        _e ->
+          Logger.info("Couldn't get object via AP, trying out OStatus fetching...")
+
+          case OStatus.fetch_activity_from_url(id) do
+            {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"])}
+            e -> e
+          end
+      end
+    end
+  end
+
+  def fetch_and_contain_remote_object_from_id(id) do
+    Logger.info("Fetching #{id} via AP")
+
+    with true <- String.starts_with?(id, "http"),
+         {:ok, %{body: body, status_code: code}} when code in 200..299 <-
+           @httpoison.get(
+             id,
+             [Accept: "application/activity+json"],
+             follow_redirect: true,
+             timeout: 10000,
+             recv_timeout: 20000
+           ),
+         {:ok, data} <- Jason.decode(body),
+         :ok <- Containment.contain_origin_from_id(id, data) do
+      {:ok, data}
+    else
+      e ->
+        {:error, e}
+    end
+  end
+end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -1,6 +1,6 @@
 defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.{Activity, Repo, Object, Upload, User, Notification}
-  alias Pleroma.Object.Containment
+  alias Pleroma.Object.Fetcher
   alias Pleroma.Web.ActivityPub.{Transmogrifier, MRF}
   alias Pleroma.Web.WebFinger
   alias Pleroma.Web.Federator
@@ -629,7 +629,7 @@ def user_data_from_user_object(data) do
   end
 
   def fetch_and_prepare_user_from_ap_id(ap_id) do
-    with {:ok, data} <- fetch_and_contain_remote_object_from_id(ap_id) do
+    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do
       user_data_from_user_object(data)
     else
       e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
@@ -723,65 +723,6 @@ def publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do
     )
   end
 
-  # TODO:
-  # This will create a Create activity, which we need internally at the moment.
-  def fetch_object_from_id(id) do
-    if object = Object.get_cached_by_ap_id(id) do
-      {:ok, object}
-    else
-      Logger.info("Fetching #{id} via AP")
-
-      with {:ok, data} <- fetch_and_contain_remote_object_from_id(id),
-           nil <- Object.normalize(data),
-           params <- %{
-             "type" => "Create",
-             "to" => data["to"],
-             "cc" => data["cc"],
-             "actor" => data["actor"] || data["attributedTo"],
-             "object" => data
-           },
-           :ok <- Containment.contain_origin(id, params),
-           {:ok, activity} <- Transmogrifier.handle_incoming(params) do
-        {:ok, Object.normalize(activity.data["object"])}
-      else
-        {:error, {:reject, nil}} ->
-          {:reject, nil}
-
-        object = %Object{} ->
-          {:ok, object}
-
-        _e ->
-          Logger.info("Couldn't get object via AP, trying out OStatus fetching...")
-
-          case OStatus.fetch_activity_from_url(id) do
-            {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"])}
-            e -> e
-          end
-      end
-    end
-  end
-
-  def fetch_and_contain_remote_object_from_id(id) do
-    Logger.info("Fetching #{id} via AP")
-
-    with true <- String.starts_with?(id, "http"),
-         {:ok, %{body: body, status_code: code}} when code in 200..299 <-
-           @httpoison.get(
-             id,
-             [Accept: "application/activity+json"],
-             follow_redirect: true,
-             timeout: 10000,
-             recv_timeout: 20000
-           ),
-         {:ok, data} <- Jason.decode(body),
-         :ok <- Containment.contain_origin_from_id(id, data) do
-      {:ok, data}
-    else
-      e ->
-        {:error, e}
-    end
-  end
-
   def is_public?(activity) do
     "https://www.w3.org/ns/activitystreams#Public" in (activity.data["to"] ++
                                                          (activity.data["cc"] || []))

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -1,6 +1,7 @@
 defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   use Pleroma.Web, :controller
   alias Pleroma.{User, Object}
+  alias Pleroma.Object.Fetcher
   alias Pleroma.Web.ActivityPub.{ObjectView, UserView}
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Relay
@@ -122,7 +123,7 @@ def inbox(conn, %{"type" => "Create"} = params) do
       "Signature missing or not from author, relayed Create message, fetching object from source"
     )
 
-    ActivityPub.fetch_object_from_id(params["object"]["id"])
+    Fetcher.fetch_object_from_id(params["object"]["id"])
 
     json(conn, "ok")
   end

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -4,7 +4,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   """
   alias Pleroma.User
   alias Pleroma.Object
-  alias Pleroma.Object.Containment
+  alias Pleroma.Object.{Containment, Fetcher}
   alias Pleroma.Activity
   alias Pleroma.Repo
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -529,8 +529,8 @@ def handle_incoming(
 
   def handle_incoming(_), do: :error
 
-  def fetch_obj_helper(id) when is_bitstring(id), do: ActivityPub.fetch_object_from_id(id)
-  def fetch_obj_helper(obj) when is_map(obj), do: ActivityPub.fetch_object_from_id(obj["id"])
+  def fetch_obj_helper(id) when is_bitstring(id), do: Fetcher.fetch_object_from_id(id)
+  def fetch_obj_helper(obj) when is_map(obj), do: Fetcher.fetch_object_from_id(obj["id"])
 
   def get_obj_helper(id) do
     if object = Object.normalize(id), do: {:ok, object}, else: nil

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -1,6 +1,7 @@
 defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   use Pleroma.Web, :controller
   alias Pleroma.{Repo, Object, Activity, User, Notification, Stats}
+  alias Pleroma.Object.Fetcher
   alias Pleroma.Web
   alias Pleroma.Web.MastodonAPI.{StatusView, AccountView, MastodonView, ListView, FilterView}
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -658,7 +659,7 @@ def unblock_domain(%{assigns: %{user: blocker}} = conn, %{"domain" => domain}) d
   def status_search(query) do
     fetched =
       if Regex.match?(~r/https?:/, query) do
-        with {:ok, object} <- ActivityPub.fetch_object_from_id(query) do
+        with {:ok, object} <- Fetcher.fetch_object_from_id(query) do
           [Activity.get_create_activity_by_object_ap_id(object.data["id"])]
         else
           _e -> []
@@ -986,7 +987,9 @@ def login(conn, %{"code" => code}) do
   def login(conn, _) do
     with {:ok, app} <- get_or_make_app() do
       path =
-        o_auth_path(conn, :authorize,
+        o_auth_path(
+          conn,
+          :authorize,
           response_type: "code",
           client_id: app.client_id,
           redirect_uri: ".",

test/web/activity_pub/activity_pub_test.exs

@@ -372,43 +372,6 @@ test "fetches the latest Follow activity" do
     end
   end
 
-  describe "fetching an object" do
-    test "it fetches an object" do
-      {:ok, object} =
-        ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
-
-      assert activity = Activity.get_create_activity_by_object_ap_id(object.data["id"])
-      assert activity.data["id"]
-
-      {:ok, object_again} =
-        ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
-
-      assert [attachment] = object.data["attachment"]
-      assert is_list(attachment["url"])
-
-      assert object == object_again
-    end
-
-    test "it works with objects only available via Ostatus" do
-      {:ok, object} = ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873")
-      assert activity = Activity.get_create_activity_by_object_ap_id(object.data["id"])
-      assert activity.data["id"]
-
-      {:ok, object_again} =
-        ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873")
-
-      assert object == object_again
-    end
-
-    test "it correctly stitches up conversations between ostatus and ap" do
-      last = "https://mstdn.io/users/mayuutann/statuses/99568293732299394"
-      {:ok, object} = ActivityPub.fetch_object_from_id(last)
-
-      object = Object.get_by_ap_id(object.data["inReplyTo"])
-      assert object
-    end
-  end
-
   describe "following / unfollowing" do
     test "creates a follow activity" do
       follower = insert(:user)
@@ -530,15 +493,6 @@ test "it filters broken threads" do
     end
   end
 
-  test "it can fetch plume articles" do
-    {:ok, object} =
-      ActivityPub.fetch_object_from_id(
-        "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"
-      )
-
-    assert object
-  end
-
   describe "update" do
     test "it creates an update activity with the new user data" do
       user = insert(:user)
@@ -560,15 +514,6 @@ test "it creates an update activity with the new user data" do
     end
   end
 
-  test "it can fetch peertube videos" do
-    {:ok, object} =
-      ActivityPub.fetch_object_from_id(
-        "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
-      )
-
-    assert object
-  end
-
   def data_uri do
     File.read!("test/fixtures/avatar_data_uri")
   end

test/web/activity_pub/transmogrifier_test.exs

@@ -894,10 +894,6 @@ test "it fixes the actor URL property to be a proper URI" do
   end
 
   describe "actor origin containment" do
-    test "it rejects objects with a bogus origin" do
-      {:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity.json")
-    end
-
     test "it rejects activities which reference objects with bogus origins" do
       data = %{
         "@context" => "https://www.w3.org/ns/activitystreams",
@@ -911,10 +907,6 @@ test "it rejects activities which reference objects with bogus origins" do
       :error = Transmogrifier.handle_incoming(data)
     end
 
-    test "it rejects objects when attributedTo is wrong (variant 1)" do
-      {:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity2.json")
-    end
-
     test "it rejects activities which reference objects that have an incorrect attribution (variant 1)" do
       data = %{
         "@context" => "https://www.w3.org/ns/activitystreams",
@@ -928,10 +920,6 @@ test "it rejects activities which reference objects that have an incorrect attri
       :error = Transmogrifier.handle_incoming(data)
     end
 
-    test "it rejects objects when attributedTo is wrong (variant 2)" do
-      {:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity3.json")
-    end
-
     test "it rejects activities which reference objects that have an incorrect attribution (variant 2)" do
       data = %{
         "@context" => "https://www.w3.org/ns/activitystreams",