Merge branch 'develop' of https://akkoma.dev/AkkomaGang/akkoma into use_fep-c16b_formatting_mfm_functions

CHANGELOG.md

@@ -6,7 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
-## BREAKING
+## Added
+
+## Fixed
+
+## Changed
+- Dropped obsolete `ap_enabled` indicator from user table and associated buggy logic
+- The remote user count in prometheus metrics is now an estimate instead of an exact number
+  since the latter proved unreasonably costly to obtain for a merely nice-to-have statistic
+- Various other tweaks improving stat query performance and avoiding unecessary work on received AP documents
 - The HTML content for new posts (both Client-to-Server as well as Server-to-Server communication) will now use a different formatting to represent MFM. See [FEP-c16b](https://codeberg.org/fediverse/fep/src/branch/main/fep/c16b/fep-c16b.md) for more details.
 
 ## 2025.01.01

FEDERATION.md

@@ -10,6 +10,7 @@
 ## Supported FEPs
 
 - [FEP-67ff: FEDERATION](https://codeberg.org/fediverse/fep/src/branch/main/fep/67ff/fep-67ff.md)
+- [FEP-dc88: Formatting Mathematics](https://codeberg.org/fediverse/fep/src/branch/main/fep/dc88/fep-dc88.md)
 - [FEP-f1d5: NodeInfo in Fediverse Software](https://codeberg.org/fediverse/fep/src/branch/main/fep/f1d5/fep-f1d5.md)
 - [FEP-fffd: Proxy Objects](https://codeberg.org/fediverse/fep/src/branch/main/fep/fffd/fep-fffd.md)
 - [FEP-c16b: Formatting MFM functions](https://codeberg.org/fediverse/fep/src/branch/main/fep/c16b/fep-c16b.md)

config/config.exs

@@ -602,7 +602,7 @@
     federator_incoming: 5,
     federator_outgoing: 5,
     search_indexing: 2,
-    rich_media_backfill: 3
+    rich_media_backfill: 1
   ],
   timeout: [
     activity_expiration: :timer.seconds(5),

lib/pleroma/collections/fetcher.ex

@@ -14,7 +14,7 @@ defmodule Akkoma.Collections.Fetcher do
   @spec fetch_collection(String.t() | map()) :: {:ok, [Pleroma.Object.t()]} | {:error, any()}
   def fetch_collection(ap_id) when is_binary(ap_id) do
     with {:ok, page} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do
-      {:ok, objects_from_collection(page)}
+      partial_as_success(objects_from_collection(page))
     else
       e ->
         Logger.error("Could not fetch collection #{ap_id} - #{inspect(e)}")
@@ -24,9 +24,12 @@ def fetch_collection(ap_id) when is_binary(ap_id) do
 
   def fetch_collection(%{"type" => type} = page)
       when type in ["Collection", "OrderedCollection", "CollectionPage", "OrderedCollectionPage"] do
-    {:ok, objects_from_collection(page)}
+    partial_as_success(objects_from_collection(page))
   end
 
+  defp partial_as_success({:partial, items}), do: {:ok, items}
+  defp partial_as_success(res), do: res
+
   defp items_in_page(%{"type" => type, "orderedItems" => items})
        when is_list(items) and type in ["OrderedCollection", "OrderedCollectionPage"],
        do: items
@@ -53,11 +56,11 @@ defp objects_from_collection(%{"type" => type, "first" => %{"id" => id}})
     fetch_page_items(id)
   end
 
-  defp objects_from_collection(_page), do: []
+  defp objects_from_collection(_page), do: {:ok, []}
 
   defp fetch_page_items(id, items \\ []) do
     if Enum.count(items) >= Config.get([:activitypub, :max_collection_objects]) do
-      items
+      {:ok, items}
     else
       with {:ok, page} <- Fetcher.fetch_and_contain_remote_object_from_id(id) do
         objects = items_in_page(page)
@@ -65,18 +68,22 @@ defp fetch_page_items(id, items \\ []) do
         if Enum.count(objects) > 0 do
           maybe_next_page(page, items ++ objects)
         else
-          items
+          {:ok, items}
         end
       else
         {:error, :not_found} ->
-          items
+          {:ok, items}
 
         {:error, :forbidden} ->
-          items
+          {:ok, items}
 
         {:error, error} ->
           Logger.error("Could not fetch page #{id} - #{inspect(error)}")
-          {:error, error}
+
+          case items do
+            [] -> {:error, error}
+            _ -> {:partial, items}
+          end
       end
     end
   end
@@ -85,5 +92,5 @@ defp maybe_next_page(%{"next" => id}, items) when is_binary(id) do
     fetch_page_items(id, items)
   end
 
-  defp maybe_next_page(_, items), do: items
+  defp maybe_next_page(_, items), do: {:ok, items}
 end

lib/pleroma/instances/instance.ex

@@ -158,6 +158,14 @@ def needs_update(%Instance{metadata_updated_at: metadata_updated_at}) do
     NaiveDateTime.diff(now, metadata_updated_at) > 86_400
   end
 
+  def needs_update(%URI{host: host}) do
+    with %Instance{} = instance <- Repo.get_by(Instance, %{host: host}) do
+      needs_update(instance)
+    else
+      _ -> true
+    end
+  end
+
   def local do
     %Instance{
       host: Pleroma.Web.Endpoint.host(),
@@ -180,7 +188,7 @@ def update_metadata(%URI{host: host} = uri) do
   defp do_update_metadata(%URI{host: host} = uri, existing_record) do
     if existing_record do
       if needs_update(existing_record) do
-        Logger.info("Updating metadata for #{host}")
+        Logger.debug("Updating metadata for #{host}")
         favicon = scrape_favicon(uri)
         nodeinfo = scrape_nodeinfo(uri)
 
@@ -199,7 +207,7 @@ defp do_update_metadata(%URI{host: host} = uri, existing_record) do
       favicon = scrape_favicon(uri)
       nodeinfo = scrape_nodeinfo(uri)
 
-      Logger.info("Creating metadata for #{host}")
+      Logger.debug("Creating metadata for #{host}")
 
       %Instance{}
       |> changeset(%{

lib/pleroma/object.ex

@@ -215,6 +215,11 @@ def get_cached_by_ap_id(ap_id) do
     end
   end
 
+  # Intentionally accepts non-Object arguments!
+  @spec is_tombstone_object?(term()) :: boolean()
+  def is_tombstone_object?(%Object{data: %{"type" => "Tombstone"}}), do: true
+  def is_tombstone_object?(_), do: false
+
   def make_tombstone(%Object{data: %{"id" => id, "type" => type}}, deleted \\ DateTime.utc_now()) do
     %ObjectTombstone{
       id: id,

lib/pleroma/search/database_search.ex

@@ -40,12 +40,6 @@ def search(user, search_query, options \\ []) do
     end
   end
 
-  @impl true
-  def add_to_index(_activity), do: nil
-
-  @impl true
-  def remove_from_index(_object), do: nil
-
   def maybe_restrict_author(query, %User{} = author) do
     Activity.Queries.by_author(query, author)
   end

lib/pleroma/search/search_backend.ex

@@ -14,4 +14,6 @@ defmodule Pleroma.Search.SearchBackend do
   from index.
   """
   @callback remove_from_index(object :: Pleroma.Object.t()) :: {:ok, any()} | {:error, any()}
+
+  @optional_callbacks add_to_index: 1, remove_from_index: 1
 end

lib/pleroma/stats.ex

@@ -10,6 +10,7 @@ defmodule Pleroma.Stats do
   alias Pleroma.CounterCache
   alias Pleroma.Repo
   alias Pleroma.User
+  alias Pleroma.Instances.Instance
 
   @interval :timer.seconds(300)
 
@@ -39,7 +40,8 @@ def force_update do
   @spec get_stats() :: %{
           domain_count: non_neg_integer(),
           status_count: non_neg_integer(),
-          user_count: non_neg_integer()
+          user_count: non_neg_integer(),
+          remote_user_count: non_neg_integer()
         }
   def get_stats do
     %{stats: stats} = GenServer.call(__MODULE__, :get_state)
@@ -60,41 +62,39 @@ def get_peers do
           stats: %{
             domain_count: non_neg_integer(),
             status_count: non_neg_integer(),
-            user_count: non_neg_integer()
+            user_count: non_neg_integer(),
+            remote_user_count: non_neg_integer()
           }
         }
   def calculate_stat_data do
+    # instances table has an unique constraint on the host column
     peers =
       from(
-        u in User,
+        i in Instance,
-        select: fragment("distinct split_part(?, '@', 2)", u.nickname),
+        select: i.host
-        where: u.local != ^true
       )
       |> Repo.all()
-      |> Enum.filter(& &1)
 
     domain_count = Enum.count(peers)
 
     status_count = Repo.aggregate(User.Query.build(%{local: true}), :sum, :note_count)
 
-    users_query =
+    # there are few enough local users for postgres to use an index scan
+    # (also here an exact count is a bit more important)
+    user_count =
       from(u in User,
         where: u.is_active == true,
         where: u.local == true,
         where: not is_nil(u.nickname),
         where: not u.invisible
       )
+      |> Repo.aggregate(:count, :id)
 
-    remote_users_query =
+    # but mostly numerous remote users leading to a full a full table scan
-      from(u in User,
+    # (ecto currently doesn't allow building queries without explicit table)
-        where: u.is_active == true,
+    %{rows: [[remote_user_count]]} =
-        where: u.local == false,
+      "SELECT estimate_remote_user_count();"
-        where: not is_nil(u.nickname),
+      |> Pleroma.Repo.query!()
-        where: not u.invisible
-      )
-
-    user_count = Repo.aggregate(users_query, :count, :id)
-    remote_user_count = Repo.aggregate(remote_users_query, :count, :id)
 
     %{
       peers: peers,

lib/pleroma/user.ex

@@ -127,7 +127,6 @@ defmodule Pleroma.User do
     field(:domain_blocks, {:array, :string}, default: [])
     field(:is_active, :boolean, default: true)
     field(:no_rich_text, :boolean, default: false)
-    field(:ap_enabled, :boolean, default: false)
     field(:is_moderator, :boolean, default: false)
     field(:is_admin, :boolean, default: false)
     field(:show_role, :boolean, default: true)
@@ -473,7 +472,6 @@ def remote_user_changeset(struct \\ %User{local: false}, params) do
         :shared_inbox,
         :nickname,
         :avatar,
-        :ap_enabled,
         :banner,
         :background,
         :is_locked,
@@ -1006,11 +1004,7 @@ def maybe_direct_follow(%User{} = follower, %User{local: true} = followed) do
   end
 
   def maybe_direct_follow(%User{} = follower, %User{} = followed) do
-    if not ap_enabled?(followed) do
+    {:ok, follower, followed}
-      follow(follower, followed)
-    else
-      {:ok, follower, followed}
-    end
   end
 
   @doc "A mass follow for local users. Respects blocks in both directions but does not create activities."
@@ -1826,7 +1820,6 @@ def purge_user_changeset(user) do
       confirmation_token: nil,
       domain_blocks: [],
       is_active: false,
-      ap_enabled: false,
       is_moderator: false,
       is_admin: false,
       mastofe_settings: nil,
@@ -2006,8 +1999,20 @@ def get_or_fetch_by_ap_id(ap_id, options \\ []) do
       {%User{} = user, _} ->
         {:ok, user}
 
-      e ->
+      {_, {:error, {:reject, :mrf}}} ->
+        Logger.debug("Rejected to fetch user due to MRF: #{ap_id}")
+        {:error, {:reject, :mrf}}
+
+      {_, {:error, :not_found}} ->
+        Logger.debug("User doesn't exist (anymore): #{ap_id}")
+        {:error, :not_found}
+
+      {_, {:error, e}} ->
         Logger.error("Could not fetch user #{ap_id}, #{inspect(e)}")
+        {:error, e}
+
+      e ->
+        Logger.error("Unexpected error condition while fetching user #{ap_id}, #{inspect(e)}")
         {:error, :not_found}
     end
   end
@@ -2073,10 +2078,6 @@ def get_public_key_for_ap_id(ap_id) do
     end
   end
 
-  def ap_enabled?(%User{local: true}), do: true
-  def ap_enabled?(%User{ap_enabled: ap_enabled}), do: ap_enabled
-  def ap_enabled?(_), do: false
-
   @doc "Gets or fetch a user by uri or nickname."
   @spec get_or_fetch(String.t()) :: {:ok, User.t()} | {:error, String.t()}
   def get_or_fetch("http://" <> _host = uri), do: get_or_fetch_by_ap_id(uri)
@@ -2580,10 +2581,10 @@ def add_pinned_object_id(%User{} = user, object_id) do
           [pinned_objects: "You have already pinned the maximum number of statuses"]
         end
       end)
+      |> update_and_set_cache()
     else
-      change(user)
+      {:ok, user}
     end
-    |> update_and_set_cache()
   end
 
   @spec remove_pinned_object_id(User.t(), String.t()) :: {:ok, t()} | {:error, term()}

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -1626,7 +1626,6 @@ defp object_to_user_data(data, additional) do
     %{
       ap_id: data["id"],
       uri: get_actor_url(data["url"]),
-      ap_enabled: true,
       banner: normalize_image(data["image"]),
       background: normalize_image(data["backgroundUrl"]),
       fields: fields,
@@ -1743,7 +1742,7 @@ def user_data_from_user_object(data, additional \\ []) do
     end
   end
 
-  def fetch_and_prepare_user_from_ap_id(ap_id, additional \\ []) do
+  defp fetch_and_prepare_user_from_ap_id(ap_id, additional) do
     with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id),
          {:valid, {:ok, _, _}} <- {:valid, UserValidator.validate(data, [])},
          {:ok, data} <- user_data_from_user_object(data, additional) do
@@ -1751,19 +1750,16 @@ def fetch_and_prepare_user_from_ap_id(ap_id, additional \\ []) do
     else
       # If this has been deleted, only log a debug and not an error
       {:error, {"Object has been deleted", _, _} = e} ->
-        Logger.debug("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
+        Logger.debug("User was explicitly deleted #{ap_id}, #{inspect(e)}")
-        {:error, e}
+        {:error, :not_found}
 
-      {:reject, reason} = e ->
+      {:reject, _reason} = e ->
-        Logger.debug("Rejected user #{ap_id}: #{inspect(reason)}")
         {:error, e}
 
       {:valid, reason} ->
-        Logger.debug("Data is not a valid user #{ap_id}: #{inspect(reason)}")
+        {:error, {:validate, reason}}
-        {:error, "Not a user"}
 
       {:error, e} ->
-        Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
         {:error, e}
     end
   end
@@ -1801,7 +1797,7 @@ def pin_data_from_featured_collection(%{
     else
       e ->
         Logger.error("Could not decode featured collection at fetch #{first}, #{inspect(e)}")
-        {:ok, %{}}
+        %{}
     end
   end
 
@@ -1811,14 +1807,18 @@ def pin_data_from_featured_collection(
         } = collection
       )
       when type in ["OrderedCollection", "Collection"] do
-    {:ok, objects} = Collections.Fetcher.fetch_collection(collection)
+    with {:ok, objects} <- Collections.Fetcher.fetch_collection(collection) do
-
+      # Items can either be a map _or_ a string
-    # Items can either be a map _or_ a string
+      objects
-    objects
+      |> Map.new(fn
-    |> Map.new(fn
+        ap_id when is_binary(ap_id) -> {ap_id, NaiveDateTime.utc_now()}
-      ap_id when is_binary(ap_id) -> {ap_id, NaiveDateTime.utc_now()}
+        %{"id" => object_ap_id} -> {object_ap_id, NaiveDateTime.utc_now()}
-      %{"id" => object_ap_id} -> {object_ap_id, NaiveDateTime.utc_now()}
+      end)
-    end)
+    else
+      e ->
+        Logger.warning("Failed to fetch featured collection #{collection}, #{inspect(e)}")
+        %{}
+    end
   end
 
   def pin_data_from_featured_collection(obj) do
@@ -1857,31 +1857,27 @@ def enqueue_pin_fetches(_), do: nil
   def make_user_from_ap_id(ap_id, additional \\ []) do
     user = User.get_cached_by_ap_id(ap_id)
 
-    if user && !User.ap_enabled?(user) do
+    with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, additional) do
-      Transmogrifier.upgrade_user_from_ap_id(ap_id)
+      user =
-    else
+        if data.ap_id != ap_id do
-      with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, additional) do
+          User.get_cached_by_ap_id(data.ap_id)
-        user =
-          if data.ap_id != ap_id do
-            User.get_cached_by_ap_id(data.ap_id)
-          else
-            user
-          end
-
-        if user do
-          user
-          |> User.remote_user_changeset(data)
-          |> User.update_and_set_cache()
-          |> tap(fn _ -> enqueue_pin_fetches(data) end)
         else
-          maybe_handle_clashing_nickname(data)
+          user
-
-          data
-          |> User.remote_user_changeset()
-          |> Repo.insert()
-          |> User.set_cache()
-          |> tap(fn _ -> enqueue_pin_fetches(data) end)
         end
+
+      if user do
+        user
+        |> User.remote_user_changeset(data)
+        |> User.update_and_set_cache()
+        |> tap(fn _ -> enqueue_pin_fetches(data) end)
+      else
+        maybe_handle_clashing_nickname(data)
+
+        data
+        |> User.remote_user_changeset()
+        |> Repo.insert()
+        |> User.set_cache()
+        |> tap(fn _ -> enqueue_pin_fetches(data) end)
       end
     end
   end

lib/pleroma/web/activity_pub/object_validator.ex

@@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Object
   alias Pleroma.Object.Containment
+  alias Pleroma.Object.Fetcher
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator
@@ -253,9 +254,28 @@ def fetch_actor(object) do
   end
 
   def fetch_actor_and_object(object) do
-    fetch_actor(object)
+    # Fetcher.fetch_object_from_id already first does a local db lookup
-    Object.normalize(object["object"], fetch: true)
+    with {:ok, %User{}} <- fetch_actor(object),
-    :ok
+         {:ap_id, id} when is_binary(id) <-
+           {:ap_id, Pleroma.Web.ActivityPub.Utils.get_ap_id(object["object"])},
+         {:ok, %Object{}} <- Fetcher.fetch_object_from_id(id) do
+      :ok
+    else
+      {:ap_id, id} ->
+        {:error, {:validate, "Invalid AP id: #{inspect(id)}"}}
+
+      # if actor: late post from a previously unknown, deleted profile
+      # if object: private post we're not allowed to access
+      # (other HTTP replies might just indicate a temporary network failure though!)
+      {:error, e} when e in [:not_found, :forbidden] ->
+        {:error, :ignore}
+
+      {:error, _} = e ->
+        e
+
+      e ->
+        {:error, e}
+    end
   end
 
   defp for_each_history_item(

lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex

@@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do
   import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
 
   require Pleroma.Constants
+  require Logger
 
   alias Pleroma.User
 
@@ -27,14 +28,21 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do
   end
 
   def cast_and_validate(data) do
-    {:ok, actor} = User.get_or_fetch_by_ap_id(data["actor"])
+    with {_, {:ok, actor}} <- {:user, User.get_or_fetch_by_ap_id(data["actor"])},
+         {_, {:ok, actor}} <- {:feataddr, maybe_refetch_user(actor)} do
+      data
+      |> maybe_fix_data_for_mastodon(actor)
+      |> cast_data()
+      |> validate_data(actor)
+    else
+      {:feataddr, _} ->
+        {:error,
+         {:validate,
+          "Actor doesn't provide featured collection address to verify against: #{data["id"]}"}}
 
-    {:ok, actor} = maybe_refetch_user(actor)
+      {:user, _} ->
-
+        {:error, :link_resolve_failed}
-    data
+    end
-    |> maybe_fix_data_for_mastodon(actor)
-    |> cast_data()
-    |> validate_data(actor)
   end
 
   defp maybe_fix_data_for_mastodon(data, actor) do
@@ -73,6 +81,9 @@ defp maybe_refetch_user(%User{featured_address: address} = user) when is_binary(
   end
 
   defp maybe_refetch_user(%User{ap_id: ap_id}) do
-    Pleroma.Web.ActivityPub.Transmogrifier.upgrade_user_from_ap_id(ap_id)
+    # If the user didn't expose a featured collection before,
+    # recheck now so we can verify perms for add/remove.
+    # But wait at least 5s to avoid rapid refetches in edge cases
+    User.get_or_fetch_by_ap_id(ap_id, maximum_age: 5)
   end
 end

lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex

@@ -71,7 +71,7 @@ defp fix_tag(data), do: Map.drop(data, ["tag"])
 
   defp fix_replies(%{"replies" => replies} = data) when is_list(replies), do: data
 
-  defp fix_replies(%{"replies" => %{"first" => first}} = data) do
+  defp fix_replies(%{"replies" => %{"first" => first}} = data) when is_binary(first) do
     with {:ok, replies} <- Akkoma.Collections.Fetcher.fetch_collection(first) do
       Map.put(data, "replies", replies)
     else
@@ -81,6 +81,10 @@ defp fix_replies(%{"replies" => %{"first" => first}} = data) do
     end
   end
 
+  defp fix_replies(%{"replies" => %{"first" => %{"items" => replies}}} = data)
+       when is_list(replies),
+       do: Map.put(data, "replies", replies)
+
   defp fix_replies(%{"replies" => %{"items" => replies}} = data) when is_list(replies),
     do: Map.put(data, "replies", replies)
 

lib/pleroma/web/activity_pub/object_validators/common_validations.ex

@@ -54,10 +54,14 @@ def validate_actor_presence(cng, options \\ []) do
   def validate_object_presence(cng, options \\ []) do
     field_name = Keyword.get(options, :field_name, :object)
     allowed_types = Keyword.get(options, :allowed_types, false)
+    allowed_categories = Keyword.get(options, :allowed_object_categores, [:object, :activity])
 
     cng
     |> validate_change(field_name, fn field_name, object_id ->
-      object = Object.get_cached_by_ap_id(object_id) || Activity.get_by_ap_id(object_id)
+      object =
+        (:object in allowed_categories && Object.get_cached_by_ap_id(object_id)) ||
+          (:activity in allowed_categories && Activity.get_by_ap_id(object_id)) ||
+          nil
 
       cond do
         !object ->

lib/pleroma/web/activity_pub/object_validators/delete_validator.ex

@@ -61,7 +61,10 @@ defp validate_data(cng) do
     |> validate_inclusion(:type, ["Delete"])
     |> validate_delete_actor(:actor)
     |> validate_modification_rights()
-    |> validate_object_or_user_presence(allowed_types: @deletable_types)
+    |> validate_object_or_user_presence(
+      allowed_types: @deletable_types,
+      allowed_object_categories: [:object]
+    )
     |> add_deleted_activity_id()
   end
 

lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex

@@ -129,7 +129,7 @@ defp validate_data(data_cng) do
     |> validate_inclusion(:type, ["EmojiReact"])
     |> validate_required([:id, :type, :object, :actor, :context, :to, :cc, :content])
     |> validate_actor_presence()
-    |> validate_object_presence()
+    |> validate_object_presence(allowed_object_categories: [:object])
     |> validate_emoji()
     |> maybe_validate_tag_presence()
   end

lib/pleroma/web/activity_pub/object_validators/like_validator.ex

@@ -66,7 +66,7 @@ defp validate_data(data_cng) do
     |> validate_inclusion(:type, ["Like"])
     |> validate_required([:id, :type, :object, :actor, :context, :to, :cc])
     |> validate_actor_presence()
-    |> validate_object_presence()
+    |> validate_object_presence(allowed_object_categories: [:object])
     |> validate_existing_like()
   end
 

lib/pleroma/web/activity_pub/object_validators/undo_validator.ex

@@ -44,7 +44,7 @@ defp validate_data(data_cng) do
     |> validate_inclusion(:type, ["Undo"])
     |> validate_required([:id, :type, :object, :actor, :to, :cc])
     |> validate_undo_actor(:actor)
-    |> validate_object_presence()
+    |> validate_object_presence(allowed_object_categories: [:activity])
     |> validate_undo_rights()
   end
 

lib/pleroma/web/activity_pub/publisher.ex

@@ -219,7 +219,6 @@ def publish(%User{} = actor, %{data: %{"bcc" => bcc}} = activity)
 
     inboxes =
       recipients
-      |> Enum.filter(&User.ap_enabled?/1)
       |> Enum.map(fn actor -> actor.inbox end)
       |> Enum.filter(fn inbox -> should_federate?(inbox) end)
       |> Instances.filter_reachable()
@@ -261,7 +260,6 @@ def publish(%User{} = actor, %Activity{} = activity) do
     json = Jason.encode!(data)
 
     recipients(actor, activity)
-    |> Enum.filter(fn user -> User.ap_enabled?(user) end)
     |> Enum.map(fn %User{} = user ->
       determine_inbox(activity, user)
     end)

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -21,7 +21,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
   alias Pleroma.Web.Federator
-  alias Pleroma.Workers.TransmogrifierWorker
 
   import Ecto.Query
 
@@ -520,10 +519,22 @@ defp handle_incoming_normalised(
 
   defp handle_incoming_normalised(%{"type" => type} = data, _options)
        when type in ~w{Like EmojiReact Announce Add Remove} do
-    with :ok <- ObjectValidator.fetch_actor_and_object(data),
+    with {_, :ok} <- {:link, ObjectValidator.fetch_actor_and_object(data)},
          {:ok, activity, _meta} <- Pipeline.common_pipeline(data, local: false) do
       {:ok, activity}
     else
+      {:link, {:error, :ignore}} ->
+        {:error, :ignore}
+
+      {:link, {:error, {:validate, _}} = e} ->
+        e
+
+      {:link, {:error, {:reject, _}} = e} ->
+        e
+
+      {:link, _} ->
+        {:error, :link_resolve_failed}
+
       e ->
         {:error, e}
     end
@@ -545,22 +556,45 @@ defp handle_incoming_normalised(
          %{"type" => "Delete"} = data,
          _options
        ) do
-    with {:ok, activity, _} <-
+    oid_result = ObjectValidators.ObjectID.cast(data["object"])
-           Pipeline.common_pipeline(data, local: false) do
+
+    with {_, {:ok, object_id}} <- {:object_id, oid_result},
+         object <- Object.get_cached_by_ap_id(object_id),
+         {_, false} <- {:tombstone, Object.is_tombstone_object?(object) && !data["actor"]},
+         {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
       {:ok, activity}
     else
-      {:error, {:validate, _}} = e ->
+      {:object_id, _} ->
-        # Check if we have a create activity for this
+        {:error, {:validate, "Invalid object id: #{data["object"]}"}}
-        with {:ok, object_id} <- ObjectValidators.ObjectID.cast(data["object"]),
+
-             %Activity{data: %{"actor" => actor}} <-
+      {:tombstone, true} ->
-               Activity.create_by_object_ap_id(object_id) |> Repo.one(),
+        {:error, :ignore}
-             # We have one, insert a tombstone and retry
+
-             {:ok, tombstone_data, _} <- Builder.tombstone(actor, object_id),
+      {:error, {:validate, {:error, %Ecto.Changeset{errors: errors}}}} = e ->
-             {:ok, _tombstone} <- Object.create(tombstone_data) do
+        if errors[:object] == {"can't find object", []} do
-          handle_incoming(data)
+          # Check if we have a create activity for this
+          # (e.g. from a db prune without --prune-activities)
+          # We'd still like to process side effects so insert a fake tombstone and retry
+          # (real tombstones from Object.delete do not have an actor field)
+          with {:ok, object_id} <- ObjectValidators.ObjectID.cast(data["object"]),
+               {_, %Activity{data: %{"actor" => actor}}} <-
+                 {:create, Activity.create_by_object_ap_id(object_id) |> Repo.one()},
+               {:ok, tombstone_data, _} <- Builder.tombstone(actor, object_id),
+               {:ok, _tombstone} <- Object.create(tombstone_data) do
+            handle_incoming(data)
+          else
+            {:create, _} -> {:error, :ignore}
+            _ -> e
+          end
         else
-          _ -> e
+          e
         end
+
+      {:error, _} = e ->
+        e
+
+      e ->
+        {:error, e}
     end
   end
 
@@ -593,6 +627,20 @@ defp handle_incoming_normalised(
        when type in ["Like", "EmojiReact", "Announce", "Block"] do
     with {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
       {:ok, activity}
+    else
+      {:error, {:validate, {:error, %Ecto.Changeset{errors: errors}}}} = e ->
+        # If we never saw the activity being undone, no need to do anything.
+        # Inspectinging the validation error content is a bit akward, but looking up the Activity
+        # ahead of time here would be too costly since Activity queries are not cached
+        # and there's no way atm to pass the retrieved result along along
+        if errors[:object] == {"can't find object", []} do
+          {:error, :ignore}
+        else
+          e
+        end
+
+      e ->
+        e
     end
   end
 
@@ -1007,47 +1055,6 @@ defp strip_internal_tags(%{"tag" => tags} = object) do
 
   defp strip_internal_tags(object), do: object
 
-  def perform(:user_upgrade, user) do
-    # we pass a fake user so that the followers collection is stripped away
-    old_follower_address = User.ap_followers(%User{nickname: user.nickname})
-
-    from(
-      a in Activity,
-      where: ^old_follower_address in a.recipients,
-      update: [
-        set: [
-          recipients:
-            fragment(
-              "array_replace(?,?,?)",
-              a.recipients,
-              ^old_follower_address,
-              ^user.follower_address
-            )
-        ]
-      ]
-    )
-    |> Repo.update_all([])
-  end
-
-  def upgrade_user_from_ap_id(ap_id) do
-    with %User{local: false} = user <- User.get_cached_by_ap_id(ap_id),
-         {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id),
-         {:ok, user} <- update_user(user, data) do
-      ActivityPub.enqueue_pin_fetches(user)
-      TransmogrifierWorker.enqueue("user_upgrade", %{"user_id" => user.id})
-      {:ok, user}
-    else
-      %User{} = user -> {:ok, user}
-      e -> e
-    end
-  end
-
-  defp update_user(user, data) do
-    user
-    |> User.remote_user_changeset(data)
-    |> User.update_and_set_cache()
-  end
-
   def maybe_fix_user_url(%{"url" => url} = data) when is_map(url) do
     Map.put(data, "url", url["href"])
   end

lib/pleroma/web/federator.ex

@@ -6,7 +6,6 @@ defmodule Pleroma.Web.Federator do
   alias Pleroma.Activity
   alias Pleroma.Object.Containment
   alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.Federator.Publisher
@@ -92,8 +91,7 @@ def perform(:incoming_ap_doc, params) do
 
     # NOTE: we use the actor ID to do the containment, this is fine because an
     # actor shouldn't be acting on objects outside their own AP server.
-    with {_, {:ok, _user}} <- {:actor, ap_enabled_actor(actor)},
+    with nil <- Activity.normalize(params["id"]),
-         nil <- Activity.normalize(params["id"]),
          {_, :ok} <-
            {:correct_origin?, Containment.contain_origin_from_id(actor, params)},
          {:ok, activity} <- Transmogrifier.handle_incoming(params) do
@@ -119,17 +117,11 @@ def perform(:incoming_ap_doc, params) do
       e ->
         # Just drop those for now
         Logger.debug(fn -> "Unhandled activity\n" <> Jason.encode!(params, pretty: true) end)
-        {:error, e}
-    end
-  end
 
-  def ap_enabled_actor(id) do
+        case e do
-    user = User.get_cached_by_ap_id(id)
+          {:error, _} -> e
-
+          _ -> {:error, e}
-    if User.ap_enabled?(user) do
+        end
-      {:ok, user}
-    else
-      ActivityPub.make_user_from_ap_id(id)
     end
   end
 end

lib/pleroma/web/rich_media/backfill.ex

@@ -57,6 +57,10 @@ def run(%{"url" => url, "url_hash" => url_hash} = args) do
         Logger.debug("Rich media error for #{url}: :content_type is #{type}")
         negative_cache(url_hash, :timer.minutes(30))
 
+      {:error, {:url, reason}} ->
+        Logger.debug("Rich media error for #{url}: refusing URL #{inspect(reason)}")
+        negative_cache(url_hash, :timer.minutes(180))
+
       e ->
         Logger.debug("Rich media error for #{url}: #{inspect(e)}")
         {:error, e}
@@ -82,7 +86,7 @@ defp maybe_schedule_expiration(url, fields) do
   end
 
   defp stream_update(%{"activity_id" => activity_id}) do
-    Logger.info("Rich media backfill: streaming update for activity #{activity_id}")
+    Logger.debug("Rich media backfill: streaming update for activity #{activity_id}")
 
     Pleroma.Activity.get_by_id(activity_id)
     |> Pleroma.Activity.normalize()

lib/pleroma/web/rich_media/parser.ex

@@ -16,12 +16,13 @@ def parse(nil), do: nil
   @spec parse(String.t()) :: {:ok, map()} | {:error, any()}
   def parse(url) do
     with {_, true} <- {:config, @config_impl.get([:rich_media, :enabled])},
-         :ok <- validate_page_url(url),
+         {_, :ok} <- {:url, validate_page_url(url)},
          {:ok, data} <- parse_url(url) do
       data = Map.put(data, "url", url)
       {:ok, data}
     else
       {:config, _} -> {:error, :rich_media_disabled}
+      {:url, {:error, reason}} -> {:error, {:url, reason}}
       e -> e
     end
   end
@@ -62,7 +63,7 @@ defp clean_parsed_data(data) do
     |> Map.new()
   end
 
-  @spec validate_page_url(URI.t() | binary()) :: :ok | :error
+  @spec validate_page_url(URI.t() | binary()) :: :ok | {:error, term()}
   defp validate_page_url(page_url) when is_binary(page_url) do
     validate_tld = @config_impl.get([Pleroma.Formatter, :validate_tld])
 
@@ -74,20 +75,20 @@ defp validate_page_url(page_url) when is_binary(page_url) do
   defp validate_page_url(%URI{host: host, scheme: "https"}) do
     cond do
       Linkify.Parser.ip?(host) ->
-        :error
+        {:error, :ip}
 
       host in @config_impl.get([:rich_media, :ignore_hosts], []) ->
-        :error
+        {:error, :ignore_hosts}
 
       get_tld(host) in @config_impl.get([:rich_media, :ignore_tld], []) ->
-        :error
+        {:error, :ignore_tld}
 
       true ->
         :ok
     end
   end
 
-  defp validate_page_url(_), do: :error
+  defp validate_page_url(_), do: {:error, "scheme mismatch"}
 
   defp parse_uri(true, url) do
     url
@@ -95,7 +96,7 @@ defp parse_uri(true, url) do
     |> validate_page_url
   end
 
-  defp parse_uri(_, _), do: :error
+  defp parse_uri(_, _), do: {:error, "not an URL"}
 
   defp get_tld(host) do
     host

lib/pleroma/web/telemetry.ex

@@ -208,8 +208,10 @@ defp summary_fallback_metrics(byte_unit \\ :byte) do
     dist_metrics ++ vm_metrics
   end
 
-  defp common_metrics do
+  defp common_metrics(byte_unit \\ :byte) do
     [
+      last_value("vm.portio.in.total", unit: {:byte, byte_unit}),
+      last_value("vm.portio.out.total", unit: {:byte, byte_unit}),
       last_value("pleroma.local_users.total"),
       last_value("pleroma.domains.total"),
       last_value("pleroma.local_statuses.total"),
@@ -220,14 +222,22 @@ defp common_metrics do
   def prometheus_metrics,
     do: common_metrics() ++ distribution_metrics() ++ summary_fallback_metrics()
 
-  def live_dashboard_metrics, do: common_metrics() ++ summary_metrics(:megabyte)
+  def live_dashboard_metrics, do: common_metrics(:megabyte) ++ summary_metrics(:megabyte)
 
   defp periodic_measurements do
     [
+      {__MODULE__, :io_stats, []},
       {__MODULE__, :instance_stats, []}
     ]
   end
 
+  def io_stats do
+    # All IO done via erlang ports, i.e. mostly network but also e.g. fasthtml_workers. NOT disk IO!
+    {{:input, input}, {:output, output}} = :erlang.statistics(:io)
+    :telemetry.execute([:vm, :portio, :in], %{total: input}, %{})
+    :telemetry.execute([:vm, :portio, :out], %{total: output}, %{})
+  end
+
   def instance_stats do
     stats = Stats.get_stats()
     :telemetry.execute([:pleroma, :local_users], %{total: stats.user_count}, %{})

lib/pleroma/web/templates/feed/feed/_author.atom.eex

@@ -11,7 +11,4 @@
   <%= if User.banner_url(@user) do %>
     <link rel="header" href="<%= User.banner_url(@user) %>"/>
   <% end %>
-  <%= if @user.local do %>
-    <ap_enabled>true</ap_enabled>
-  <% end %>
 </author>

lib/pleroma/web/templates/feed/feed/_author.rss.eex

@@ -11,7 +11,4 @@
   <%= if User.banner_url(@user) do %>
     <link rel="header"><%= User.banner_url(@user) %></link>
   <% end %>
-  <%= if @user.local do %>
-    <ap_enabled>true</ap_enabled>
-  <% end %>
 </managingEditor>

lib/pleroma/web/templates/feed/feed/_tag_author.atom.eex

@@ -8,9 +8,6 @@
     <%= if User.banner_url(@actor) do %>
       <link rel="header" href="<%= User.banner_url(@actor) %>"/>
     <% end %>
-    <%= if @actor.local do %>
-      <ap_enabled>true</ap_enabled>
-    <% end %>
   
     <poco:preferredUsername><%= @actor.nickname %></poco:preferredUsername>        
     <poco:displayName><%= @actor.name %></poco:displayName>

lib/pleroma/web/templates/static_fe/static_fe/_attachment.html.eex

@@ -1,4 +1,4 @@
-<a class="attachment" href="<%= @url %>" alt=<%= @name %>" title="<%= @name %>">
+<a class="attachment" href="<%= @url %>" alt="<%= @name %>" title="<%= @name %>">
   <%= if @nsfw do %>
   <div class="nsfw-banner">
     <div><%= gettext("Hover to show content") %></div>

lib/pleroma/workers/nodeinfo_fetcher_worker.ex

@@ -1,9 +1,30 @@
 defmodule Pleroma.Workers.NodeInfoFetcherWorker do
-  use Pleroma.Workers.WorkerHelper, queue: "nodeinfo_fetcher"
+  use Pleroma.Workers.WorkerHelper,
+    queue: "nodeinfo_fetcher",
+    unique: [
+      keys: [:op, :source_url],
+      # old jobs still get pruned after a short while
+      period: :infinity,
+      states: Oban.Job.states()
+    ]
 
   alias Oban.Job
   alias Pleroma.Instances.Instance
 
+  def enqueue(op, %{"source_url" => ap_id} = params, worker_args) do
+    # reduce to base url to avoid enqueueing unneccessary duplicates
+    domain =
+      ap_id
+      |> URI.parse()
+      |> URI.merge("/")
+
+    if Instance.needs_update(domain) do
+      do_enqueue(op, %{params | "source_url" => URI.to_string(domain)}, worker_args)
+    else
+      :ok
+    end
+  end
+
   @impl Oban.Worker
   def perform(%Job{
         args: %{"op" => "process", "source_url" => domain}

lib/pleroma/workers/receiver_worker.ex

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Workers.ReceiverWorker do
+  require Logger
+
   alias Pleroma.Web.Federator
 
   use Pleroma.Workers.WorkerHelper, queue: "federator_incoming"
@@ -12,10 +14,49 @@ def perform(%Job{args: %{"op" => "incoming_ap_doc", "params" => params}}) do
     with {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do
       {:ok, res}
     else
-      {:error, :origin_containment_failed} -> {:discard, :origin_containment_failed}
+      {:error, :origin_containment_failed} ->
-      {:error, {:reject, reason}} -> {:discard, reason}
+        {:discard, :origin_containment_failed}
-      {:error, _} = e -> e
+
-      e -> {:error, e}
+      {:error, {:reject, reason}} ->
+        {:discard, reason}
+
+      {:error, :already_present} ->
+        {:discard, :already_present}
+
+      {:error, :ignore} ->
+        {:discard, :ignore}
+
+      # invalid data or e.g. deleting an object we don't know about anyway
+      {:error, {:validate, issue}} ->
+        Logger.info("Received invalid AP document: #{inspect(issue)}")
+        {:discard, :invalid}
+
+      # rarer, but sometimes there’s an additional :error in front
+      {:error, {:error, {:validate, issue}}} ->
+        Logger.info("Received invalid AP document: (2e) #{inspect(issue)}")
+        {:discard, :invalid}
+
+      # failed to resolve a necessary referenced remote AP object;
+      # might be temporary server/network trouble thus reattempt
+      {:error, :link_resolve_failed} = e ->
+        Logger.info("Failed to resolve AP link; may retry: #{inspect(params)}")
+        e
+
+      {:error, _} = e ->
+        Logger.error("Unexpected AP doc error: #{inspect(e)} from #{inspect(params)}")
+        e
+
+      e ->
+        Logger.error("Unexpected AP doc error: (raw) #{inspect(e)} from #{inspect(params)}")
+        {:error, e}
     end
+  rescue
+    err ->
+      Logger.error(
+        "Receiver worker CRASH on #{inspect(params)} with: #{Exception.format(:error, err, __STACKTRACE__)}"
+      )
+
+      # reraise to let oban handle transaction conflicts without deductig an attempt
+      reraise err, __STACKTRACE__
   end
 end

lib/pleroma/workers/search_indexing_worker.ex

@@ -1,23 +1,38 @@
 defmodule Pleroma.Workers.SearchIndexingWorker do
   use Pleroma.Workers.WorkerHelper, queue: "search_indexing"
 
-  @impl Oban.Worker
+  defp search_module(), do: Pleroma.Config.get!([Pleroma.Search, :module])
 
+  def enqueue("add_to_index", params, worker_args) do
+    if Kernel.function_exported?(search_module(), :add_to_index, 1) do
+      do_enqueue("add_to_index", params, worker_args)
+    else
+      # XXX: or {:ok, nil} to more closely match Oban.inset()'s {:ok, job}?
+      #      or similar to unique coflict: %Oban.Job{conflict?: true} (but omitting all other fileds...)
+      :ok
+    end
+  end
+
+  def enqueue("remove_from_index", params, worker_args) do
+    if Kernel.function_exported?(search_module(), :remove_from_index, 1) do
+      do_enqueue("remove_from_index", params, worker_args)
+    else
+      :ok
+    end
+  end
+
+  @impl Oban.Worker
   def perform(%Job{args: %{"op" => "add_to_index", "activity" => activity_id}}) do
     activity = Pleroma.Activity.get_by_id_with_object(activity_id)
 
-    search_module = Pleroma.Config.get([Pleroma.Search, :module])
+    search_module().add_to_index(activity)
-
-    search_module.add_to_index(activity)
 
     :ok
   end
 
   def perform(%Job{args: %{"op" => "remove_from_index", "object" => object_id}}) do
-    search_module = Pleroma.Config.get([Pleroma.Search, :module])
-
     # Fake the object so we can remove it from the index without having to keep it in the DB
-    search_module.remove_from_index(%Pleroma.Object{id: object_id})
+    search_module().remove_from_index(%Pleroma.Object{id: object_id})
 
     :ok
   end

lib/pleroma/workers/transmogrifier_worker.ex

@@ -1,15 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Workers.TransmogrifierWorker do
-  alias Pleroma.User
-
-  use Pleroma.Workers.WorkerHelper, queue: "transmogrifier"
-
-  @impl Oban.Worker
-  def perform(%Job{args: %{"op" => "user_upgrade", "user_id" => user_id}}) do
-    user = User.get_cached_by_id(user_id)
-    Pleroma.Web.ActivityPub.Transmogrifier.perform(:user_upgrade, user)
-  end
-end

lib/pleroma/workers/worker_helper.ex

@@ -38,7 +38,7 @@ defmacro __using__(opts) do
 
       alias Oban.Job
 
-      def enqueue(op, params, worker_args \\ []) do
+      defp do_enqueue(op, params, worker_args \\ []) do
         params = Map.merge(%{"op" => op}, params)
         queue_atom = String.to_atom(unquote(queue))
         worker_args = worker_args ++ WorkerHelper.worker_args(queue_atom)
@@ -48,11 +48,16 @@ def enqueue(op, params, worker_args \\ []) do
         |> Oban.insert()
       end
 
+      def enqueue(op, params, worker_args \\ []),
+        do: do_enqueue(op, params, worker_args)
+
       @impl Oban.Worker
       def timeout(_job) do
         queue_atom = String.to_atom(unquote(queue))
         Config.get([:workers, :timeout, queue_atom], :timer.minutes(1))
       end
+
+      defoverridable enqueue: 3
     end
   end
 end

priv/repo/migrations/20241211000000_remote_user_count_estimate_function.exs

@@ -0,0 +1,38 @@
+# Akkoma: Magically expressive social media
+# Copyright © 2024 Akkoma Authors <https://akkoma.dev/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Repo.Migrations.RemoteUserCountEstimateFunction do
+  use Ecto.Migration
+
+  @function_name "estimate_remote_user_count"
+
+  def up() do
+    # yep, this EXPLAIN (ab)use is blessed by the PostgreSQL wiki:
+    #   https://wiki.postgresql.org/wiki/Count_estimate
+    """
+    CREATE OR REPLACE FUNCTION #{@function_name}()
+    RETURNS integer
+    LANGUAGE plpgsql AS $$
+      DECLARE plan jsonb;
+      BEGIN
+        EXECUTE '
+          EXPLAIN (FORMAT JSON)
+          SELECT *
+          FROM public.users
+          WHERE local = false AND
+                is_active = true AND
+                invisible = false AND
+                nickname IS NOT NULL;
+        ' INTO plan;
+        RETURN plan->0->'Plan'->'Plan Rows';
+      END;
+    $$;
+    """
+    |> execute()
+  end
+
+  def down() do
+    execute("DROP FUNCTION IF EXISTS #{@function_name}()")
+  end
+end

priv/repo/migrations/20241213000000_remove_user_ap_enabled.exs

@@ -0,0 +1,13 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2023 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Repo.Migrations.RemoveUserApEnabled do
+  use Ecto.Migration
+
+  def change do
+    alter table(:users) do
+      remove(:ap_enabled, :boolean, default: false, null: false)
+    end
+  end
+end

test/pleroma/user_test.exs

@@ -1694,7 +1694,6 @@ test "delete/1 purges a user when they wouldn't be fully deleted" do
         confirmation_token: "qqqq",
         domain_blocks: ["lain.com"],
         is_active: false,
-        ap_enabled: true,
         is_moderator: true,
         is_admin: true,
         mastofe_settings: %{"a" => "b"},
@@ -1734,7 +1733,6 @@ test "delete/1 purges a user when they wouldn't be fully deleted" do
              confirmation_token: nil,
              domain_blocks: [],
              is_active: false,
-             ap_enabled: false,
              is_moderator: false,
              is_admin: false,
              mastofe_settings: nil,
@@ -2217,8 +2215,7 @@ test "updates the counters normally on following/getting a follow when disabled"
         insert(:user,
           local: false,
           follower_address: "http://remote.org/users/masto_closed/followers",
-          following_address: "http://remote.org/users/masto_closed/following",
+          following_address: "http://remote.org/users/masto_closed/following"
-          ap_enabled: true
         )
 
       assert other_user.following_count == 0
@@ -2239,8 +2236,7 @@ test "synchronizes the counters with the remote instance for the followed when e
         insert(:user,
           local: false,
           follower_address: "http://remote.org/users/masto_closed/followers",
-          following_address: "http://remote.org/users/masto_closed/following",
+          following_address: "http://remote.org/users/masto_closed/following"
-          ap_enabled: true
         )
 
       assert other_user.following_count == 0
@@ -2261,8 +2257,7 @@ test "synchronizes the counters with the remote instance for the follower when e
         insert(:user,
           local: false,
           follower_address: "http://remote.org/users/masto_closed/followers",
-          following_address: "http://remote.org/users/masto_closed/following",
+          following_address: "http://remote.org/users/masto_closed/following"
-          ap_enabled: true
         )
 
       assert other_user.following_count == 0

test/pleroma/web/activity_pub/activity_pub_controller_test.exs

@@ -579,7 +579,6 @@ test "it inserts an incoming activity into the database" <>
       user =
         insert(:user,
           ap_id: "https://mastodon.example.org/users/raymoo",
-          ap_enabled: true,
           local: false,
           last_refreshed_at: nil
         )

test/pleroma/web/activity_pub/activity_pub_test.exs

@@ -178,7 +178,6 @@ test "it returns a user" do
       {:ok, user} = ActivityPub.make_user_from_ap_id(user_id)
       assert user.ap_id == user_id
       assert user.nickname == "admin@mastodon.example.org"
-      assert user.ap_enabled
       assert user.follower_address == "http://mastodon.example.org/users/admin/followers"
     end
 

test/pleroma/web/activity_pub/mrf/anti_link_spam_policy_test.exs

@@ -241,11 +241,11 @@ test "it rejects posts without links" do
 
       assert capture_log(fn ->
                {:reject, _} = AntiLinkSpamPolicy.filter(message)
-             end) =~ "[error] Could not decode user at fetch http://invalid.actor"
+             end) =~ "[error] Could not fetch user http://invalid.actor,"
 
       assert capture_log(fn ->
                {:reject, _} = AntiLinkSpamPolicy.filter(update_message)
-             end) =~ "[error] Could not decode user at fetch http://invalid.actor"
+             end) =~ "[error] Could not fetch user http://invalid.actor,"
     end
 
     test "it rejects posts with links" do
@@ -259,11 +259,11 @@ test "it rejects posts with links" do
 
       assert capture_log(fn ->
                {:reject, _} = AntiLinkSpamPolicy.filter(message)
-             end) =~ "[error] Could not decode user at fetch http://invalid.actor"
+             end) =~ "[error] Could not fetch user http://invalid.actor,"
 
       assert capture_log(fn ->
                {:reject, _} = AntiLinkSpamPolicy.filter(update_message)
-             end) =~ "[error] Could not decode user at fetch http://invalid.actor"
+             end) =~ "[error] Could not fetch user http://invalid.actor,"
     end
   end
 

test/pleroma/web/activity_pub/publisher_test.exs

@@ -306,15 +306,13 @@ test "publish to url with with different ports" do
       follower =
         insert(:user, %{
           local: false,
-          inbox: "https://domain.com/users/nick1/inbox",
+          inbox: "https://domain.com/users/nick1/inbox"
-          ap_enabled: true
         })
 
       another_follower =
         insert(:user, %{
           local: false,
-          inbox: "https://rejected.com/users/nick2/inbox",
+          inbox: "https://rejected.com/users/nick2/inbox"
-          ap_enabled: true
         })
 
       actor =
@@ -386,8 +384,7 @@ test "publish to url with with different ports" do
       follower =
         insert(:user, %{
           local: false,
-          inbox: "https://domain.com/users/nick1/inbox",
+          inbox: "https://domain.com/users/nick1/inbox"
-          ap_enabled: true
         })
 
       actor =
@@ -425,8 +422,7 @@ test "publish to url with with different ports" do
       follower =
         insert(:user, %{
           local: false,
-          inbox: "https://domain.com/users/nick1/inbox",
+          inbox: "https://domain.com/users/nick1/inbox"
-          ap_enabled: true
         })
 
       actor = insert(:user, follower_address: follower.ap_id)
@@ -461,15 +457,13 @@ test "publish to url with with different ports" do
       fetcher =
         insert(:user,
           local: false,
-          inbox: "https://domain.com/users/nick1/inbox",
+          inbox: "https://domain.com/users/nick1/inbox"
-          ap_enabled: true
         )
 
       another_fetcher =
         insert(:user,
           local: false,
-          inbox: "https://domain2.com/users/nick1/inbox",
+          inbox: "https://domain2.com/users/nick1/inbox"
-          ap_enabled: true
         )
 
       actor = insert(:user)

test/pleroma/web/activity_pub/relay_test.exs

@@ -29,7 +29,7 @@ test "relay actor is invisible" do
     test "returns errors when user not found" do
       assert capture_log(fn ->
                {:error, _} = Relay.follow("test-ap-id")
-             end) =~ "Could not decode user at fetch"
+             end) =~ "Could not fetch user test-ap-id,"
     end
 
     test "returns activity" do
@@ -48,7 +48,7 @@ test "returns activity" do
     test "returns errors when user not found" do
       assert capture_log(fn ->
                {:error, _} = Relay.unfollow("test-ap-id")
-             end) =~ "Could not decode user at fetch"
+             end) =~ "Could not fetch user test-ap-id,"
     end
 
     test "returns activity" do

test/pleroma/web/activity_pub/side_effects_test.exs

@@ -46,7 +46,7 @@ test "it queues a fetch of instance information" do
 
       assert_enqueued(
         worker: Pleroma.Workers.NodeInfoFetcherWorker,
-        args: %{"op" => "process", "source_url" => "https://wowee.example.com/users/1"}
+        args: %{"op" => "process", "source_url" => "https://wowee.example.com/"}
       )
     end
   end

test/pleroma/web/activity_pub/transmogrifier/add_remove_handling_test.exs

@@ -102,6 +102,7 @@ test "Add/Remove activities for remote users without featured address" do
     user =
       user
       |> Ecto.Changeset.change(featured_address: nil)
+      |> Ecto.Changeset.change(last_refreshed_at: ~N[2013-03-14 11:50:00.000000])
       |> Repo.update!()
 
     %{host: host} = URI.parse(user.ap_id)

test/pleroma/web/activity_pub/transmogrifier_test.exs

@@ -8,7 +8,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
   @moduletag :mocked
   alias Pleroma.Activity
   alias Pleroma.Object
-  alias Pleroma.Tests.ObanHelpers
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.ActivityPub.Utils
@@ -53,6 +52,25 @@ test "it works for incoming unfollows with an existing follow" do
       refute User.following?(User.get_cached_by_ap_id(data["actor"]), user)
     end
 
+    test "it ignores Undo activities for unknown objects" do
+      undo_data = %{
+        "id" => "https://remote.com/undo",
+        "type" => "Undo",
+        "actor" => "https:://remote.com/users/unknown",
+        "object" => %{
+          "id" => "https://remote.com/undone_activity/unknown",
+          "type" => "Like"
+        }
+      }
+
+      assert {:error, :ignore} == Transmogrifier.handle_incoming(undo_data)
+
+      user = insert(:user, local: false, ap_id: "https://remote.com/users/known")
+      undo_data = %{undo_data | "actor" => user.ap_id}
+
+      assert {:error, :ignore} == Transmogrifier.handle_incoming(undo_data)
+    end
+
     test "it accepts Flag activities" do
       user = insert(:user)
       other_user = insert(:user)
@@ -348,69 +366,6 @@ test "Updates of Notes are handled" do
     end
   end
 
-  describe "user upgrade" do
-    test "it upgrades a user to activitypub" do
-      user =
-        insert(:user, %{
-          nickname: "rye@niu.moe",
-          local: false,
-          ap_id: "https://niu.moe/users/rye",
-          follower_address: User.ap_followers(%User{nickname: "rye@niu.moe"})
-        })
-
-      user_two = insert(:user)
-      Pleroma.FollowingRelationship.follow(user_two, user, :follow_accept)
-
-      {:ok, activity} = CommonAPI.post(user, %{status: "test"})
-      {:ok, unrelated_activity} = CommonAPI.post(user_two, %{status: "test"})
-      assert "http://localhost:4001/users/rye@niu.moe/followers" in activity.recipients
-
-      user = User.get_cached_by_id(user.id)
-      assert user.note_count == 1
-
-      {:ok, user} = Transmogrifier.upgrade_user_from_ap_id("https://niu.moe/users/rye")
-      ObanHelpers.perform_all()
-
-      assert user.ap_enabled
-      assert user.note_count == 1
-      assert user.follower_address == "https://niu.moe/users/rye/followers"
-      assert user.following_address == "https://niu.moe/users/rye/following"
-
-      user = User.get_cached_by_id(user.id)
-      assert user.note_count == 1
-
-      activity = Activity.get_by_id(activity.id)
-      assert user.follower_address in activity.recipients
-
-      assert %{
-               "url" => [
-                 %{
-                   "href" =>
-                     "https://cdn.niu.moe/accounts/avatars/000/033/323/original/fd7f8ae0b3ffedc9.jpeg"
-                 }
-               ]
-             } = user.avatar
-
-      assert %{
-               "url" => [
-                 %{
-                   "href" =>
-                     "https://cdn.niu.moe/accounts/headers/000/033/323/original/850b3448fa5fd477.png"
-                 }
-               ]
-             } = user.banner
-
-      refute "..." in activity.recipients
-
-      unrelated_activity = Activity.get_by_id(unrelated_activity.id)
-      refute user.follower_address in unrelated_activity.recipients
-
-      user_two = User.get_cached_by_id(user_two.id)
-      assert User.following?(user_two, user)
-      refute "..." in User.following(user_two)
-    end
-  end
-
   describe "actor rewriting" do
     test "it fixes the actor URL property to be a proper URI" do
       data = %{

test/pleroma/web/common_api_test.exs

@@ -1129,7 +1129,7 @@ test "removes a pending follow for a local user" do
 
     test "cancels a pending follow for a remote user" do
       follower = insert(:user)
-      followed = insert(:user, is_locked: true, local: false, ap_enabled: true)
+      followed = insert(:user, is_locked: true, local: false)
 
       assert {:ok, follower, followed, %{id: _activity_id, data: %{"state" => "pending"}}} =
                CommonAPI.follow(follower, followed)

test/pleroma/web/federator_test.exs

@@ -79,16 +79,14 @@ test "it federates only to reachable instances via AP" do
         local: false,
         nickname: "nick1@domain.com",
         ap_id: "https://domain.com/users/nick1",
-        inbox: inbox1,
+        inbox: inbox1
-        ap_enabled: true
       })
 
       insert(:user, %{
         local: false,
         nickname: "nick2@domain2.com",
         ap_id: "https://domain2.com/users/nick2",
-        inbox: inbox2,
+        inbox: inbox2
-        ap_enabled: true
       })
 
       dt = NaiveDateTime.utc_now()
@@ -134,7 +132,7 @@ test "successfully processes incoming AP docs with correct origin" do
       assert {:ok, _activity} = ObanHelpers.perform(job)
 
       assert {:ok, job} = Federator.incoming_ap_doc(params)
-      assert {:error, :already_present} = ObanHelpers.perform(job)
+      assert {:discard, :already_present} = ObanHelpers.perform(job)
     end
 
     test "successfully normalises public scope descriptors" do

test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs

@@ -61,7 +61,9 @@ test "get instance stats", %{conn: conn} do
     {:ok, _user2} = User.set_activation(user2, false)
 
     insert(:user, %{local: false, nickname: "u@peer1.com"})
+    insert(:instance, %{domain: "peer1.com"})
     insert(:user, %{local: false, nickname: "u@peer2.com"})
+    insert(:instance, %{domain: "peer2.com"})
 
     {:ok, _} = Pleroma.Web.CommonAPI.post(user, %{status: "cofe"})
 
@@ -81,7 +83,9 @@ test "get instance stats", %{conn: conn} do
 
   test "get peers", %{conn: conn} do
     insert(:user, %{local: false, nickname: "u@peer1.com"})
+    insert(:instance, %{domain: "peer1.com"})
     insert(:user, %{local: false, nickname: "u@peer2.com"})
+    insert(:instance, %{domain: "peer2.com"})
 
     Pleroma.Stats.force_update()
 

test/pleroma/web/rich_media/parser_test.exs

@@ -109,25 +109,40 @@ test "does a HEAD request to check if the body is html" do
 
   test "refuses to crawl incomplete URLs" do
     url = "example.com/ogp"
-    assert :error == Parser.parse(url)
+    assert {:error, {:url, "scheme mismatch"}} == Parser.parse(url)
+  end
+
+  test "refuses to crawl plain HTTP and other scheme URL" do
+    [
+      "http://example.com/ogp",
+      "ftp://example.org/dist/"
+    ]
+    |> Enum.each(fn url ->
+      res = Parser.parse(url)
+
+      assert {:error, {:url, "scheme mismatch"}} == res or
+               {:error, {:url, "not an URL"}} == res
+    end)
   end
 
   test "refuses to crawl malformed URLs" do
     url = "example.com[]/ogp"
-    assert :error == Parser.parse(url)
+    assert {:error, {:url, "not an URL"}} == Parser.parse(url)
   end
 
   test "refuses to crawl URLs of private network from posts" do
     [
-      "http://127.0.0.1:4000/notice/9kCP7VNyPJXFOXDrgO",
+      "https://127.0.0.1:4000/notice/9kCP7VNyPJXFOXDrgO",
       "https://10.111.10.1/notice/9kCP7V",
       "https://172.16.32.40/notice/9kCP7V",
-      "https://192.168.10.40/notice/9kCP7V",
+      "https://192.168.10.40/notice/9kCP7V"
-      "https://pleroma.local/notice/9kCP7V"
     ]
     |> Enum.each(fn url ->
-      assert :error == Parser.parse(url)
+      assert {:error, {:url, :ip}} == Parser.parse(url)
     end)
+
+    url = "https://pleroma.local/notice/9kCP7V"
+    assert {:error, {:url, :ignore_tld}} == Parser.parse(url)
   end
 
   test "returns error when disabled" do

test/pleroma/web/twitter_api/remote_follow_controller_test.exs

@@ -132,7 +132,7 @@ test "show follow page with error when user can not be fetched by `acct` link",
                  |> html_response(200)
 
                assert response =~ "Error fetching user"
-             end) =~ ":not_found"
+             end) =~ "User doesn't exist (anymore): https://mastodon.social/users/not_found"
     end
   end
 

test/support/factory.ex

@@ -62,8 +62,7 @@ def user_factory(attrs \\ %{}) do
       last_digest_emailed_at: NaiveDateTime.utc_now(),
       last_refreshed_at: NaiveDateTime.utc_now(),
       notification_settings: %Pleroma.User.NotificationSetting{},
-      multi_factor_authentication_settings: %Pleroma.MFA.Settings{},
+      multi_factor_authentication_settings: %Pleroma.MFA.Settings{}
-      ap_enabled: true
     }
 
     urls =