Merge pull request 'Tweak users database indexes and drop exclude_visibilities' (#1019) from Oneric/akkoma:db-index-tweaks into develop

Reviewed-on: #1019

CHANGELOG.md

@@ -6,16 +6,55 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 ### REMOVED
+- DEPRECATE `config :pleroma, :instance, skip_thread_containment: false`.
+  It is due to be removed in one of the next releases if no strong arguments for keeping it are brought up.
+  It is already semi-broken for large threads and conflicts with pending optimisation and cleanup work.
+- support for `exclude_visibilities` in timeline and notification endpoints has been dropped
+- support for list visibility / list addressing has been dropped due to lack of usage, maintenance burden and redundancy with the still supported explicit-addressing feature
+- support for conversations addressing has been dropped due to lack of usage, maintenance burden and being mostly redundant with explicit addressing
+- per-visibility status counters have been dropped from `/api/v1/pleroma/admin/stats`
+  due to unreasonably perf costs added on most database operations.
+  For now, the response still contains the fields, but with stubbed-out values.
 
 ### Added
 - status responses include two new fields for ActivityPub cross-referencing: `akkoma.quote_apid` and `akkoma.in_reply_to_apid`
+- attempting to reply to an already deleted post will return an error
+  (in akkoma-fe the error will be shown and your draft message retained so you can decide
+   for yourself whether to discard it or copy and repost as a, now intentional, new thread)
+- the notification endpoint now supports the `types` parameter for filtering added in vanilla Mastodon
+- the mute endpoint now supports the `duration` parameter added in vanilla Mastodon
+  (fixes temporary mutes created via e.g. Husky)
 
 ### Fixed
 - replies and quotes to unresolvable posts now fill out IDs for replied to
   status, user or quoted status with a 404-ing ID to make them recognisable as
   replies/quotes instead of pretending they’re root posts
+- querying a status using the ID of a non-post AP activity no longer displays
+  a duplicate of the post referenced by said activity with mangled author information
+- fix users being able to interact (like, emoji react, ...) with posts they cannot access
+- fix AP fetches of local non-Create, non-Undo activities exposing the raw, unsanitised content of the referenced object
+- the above two combined allowed local users to gain access to private posts
+  of user they do not follow, but follow a follower of the author.
+  (remote users and other scenarios were to our knowledge not able to achieve this due to other restrictions)
+- fix RSS and Atom feeds of hashtag timelines potentially exposing more information than Mastodon API when restricting unauthenticated API access
+- fix mentioning and sending DMs to users with non-ASCII-alphanumerical usernames
+- correctly hide and show inlined fallback links for quotes from Mastodon instances
+- API requests with multiple unsupported parameters now will ignore all of them up to a certain limit.
+  If there are too many unsupported parameters this is indicated in the returned error message.
+- expose generic type of attachment via Masto API if remote did not send a full MIME type but indicated a generic one
+  (the \*oma-specific full mime type field in the API response remains generic however, since we don't have this info)
+- add back the default banner image we advertise in Masto API
+- correctly redirect `/users/:nickname.rss` to the RSS instead of Atom feed
 
 ### Changed
+- depreacted the `included_types` parameter in the notification endpoint; replaced by `types`
+- depreacted the `expires_in` parameter in the mute endpoint; replaced by `duration`
+- optimised emoji addition and removal
+- emoji reloading now happens asynchronously so you won't run into timeout issues with many emoji and/or a slow disk
+- upgraded all of our dependencies; this should reduce issues when running akoma with OTP28
+- prefer "summary" over "name" for the attachment alt text of incoming ActivityPub documents;
+  this fixes alt text federation from GtS and Honk
+- slightly improve index overhead for the users table
 
 
 ## 2025.10

lib/mix/tasks/pleroma/notification_settings.ex

@@ -43,6 +43,7 @@ defmodule Mix.Tasks.Pleroma.NotificationSettings do
   defp build_query(hide_notification_contents, options) do
     query =
       from(u in Pleroma.User,
+        where: u.local,
         update: [
           set: [
             notification_settings:

lib/pleroma/web/api_spec/operations/account_operation.ex

@@ -138,12 +138,6 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
           ),
           Operation.parameter(:exclude_reblogs, :query, BooleanLike, "Exclude reblogs"),
           Operation.parameter(:exclude_replies, :query, BooleanLike, "Exclude replies"),
-          Operation.parameter(
-            :exclude_visibilities,
-            :query,
-            %Schema{type: :array, items: VisibilityScope},
-            "Exclude visibilities"
-          ),
           Operation.parameter(
             :with_muted,
             :query,

lib/pleroma/web/api_spec/operations/notification_operation.ex

@@ -10,7 +10,6 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do
   alias Pleroma.Web.ApiSpec.Schemas.ApiError
   alias Pleroma.Web.ApiSpec.Schemas.BooleanLike
   alias Pleroma.Web.ApiSpec.Schemas.Status
-  alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope
 
   import Pleroma.Web.ApiSpec.Helpers
 
@@ -41,12 +40,6 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do
             %Schema{type: :string},
             "Return only notifications received from this account"
           ),
-          Operation.parameter(
-            :exclude_visibilities,
-            :query,
-            %Schema{type: :array, items: VisibilityScope},
-            "Exclude the notifications for activities with the given visibilities"
-          ),
           Operation.parameter(
             :include_types,
             :query,

lib/pleroma/web/api_spec/operations/timeline_operation.ex

@@ -8,7 +8,6 @@ defmodule Pleroma.Web.ApiSpec.TimelineOperation do
   alias Pleroma.Web.ApiSpec.Schemas.ApiError
   alias Pleroma.Web.ApiSpec.Schemas.BooleanLike
   alias Pleroma.Web.ApiSpec.Schemas.Status
-  alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope
 
   import Pleroma.Web.ApiSpec.Helpers
 
@@ -28,7 +27,6 @@ defmodule Pleroma.Web.ApiSpec.TimelineOperation do
         remote_param(),
         only_media_param(),
         with_muted_param(),
-        exclude_visibilities_param(),
         reply_visibility_param() | pagination_params()
       ],
       operationId: "TimelineController.home",
@@ -64,7 +62,6 @@ defmodule Pleroma.Web.ApiSpec.TimelineOperation do
         only_media_param(),
         remote_param(),
         with_muted_param(),
-        exclude_visibilities_param(),
         reply_visibility_param() | pagination_params()
       ],
       operationId: "TimelineController.public",
@@ -85,7 +82,6 @@ defmodule Pleroma.Web.ApiSpec.TimelineOperation do
         only_media_param(),
         remote_param(),
         with_muted_param(),
-        exclude_visibilities_param(),
         reply_visibility_param() | pagination_params()
       ],
       operationId: "TimelineController.bubble",
@@ -131,8 +127,7 @@ defmodule Pleroma.Web.ApiSpec.TimelineOperation do
         local_param(),
         only_media_param(),
         remote_param(),
-        with_muted_param(),
-        exclude_visibilities_param() | pagination_params()
+        with_muted_param()
       ],
       operationId: "TimelineController.hashtag",
       responses: %{
@@ -159,8 +154,8 @@ defmodule Pleroma.Web.ApiSpec.TimelineOperation do
         with_muted_param(),
         local_param(),
         remote_param(),
-        only_media_param(),
-        exclude_visibilities_param() | pagination_params()
+        only_media_param()
+        | pagination_params()
       ],
       operationId: "TimelineController.list",
       responses: %{
@@ -200,15 +195,6 @@ defmodule Pleroma.Web.ApiSpec.TimelineOperation do
     Operation.parameter(:with_muted, :query, BooleanLike, "Include activities by muted users")
   end
 
-  defp exclude_visibilities_param do
-    Operation.parameter(
-      :exclude_visibilities,
-      :query,
-      %Schema{type: :array, items: VisibilityScope},
-      "Exclude the statuses with the given visibilities"
-    )
-  end
-
   defp reply_visibility_param do
     Operation.parameter(
       :reply_visibility,

lib/pleroma/web/mastodon_api/mastodon_api.ex

@@ -81,7 +81,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do
     param_types = %{
       exclude_types: {:array, :string},
       types: {:array, :string},
-      exclude_visibilities: {:array, :string},
+      # exclude_visibilities: {:array, :string},
       reblogs: :boolean,
       with_muted: :boolean,
       account_ap_id: :string,

lib/pleroma/workers/cron/digest_emails_worker.ex

@@ -31,6 +31,7 @@ defmodule Pleroma.Workers.Cron.DigestEmailsWorker do
 
       from(u in inactive_users_query,
         where: fragment(~s(? ->'digest' @> 'true'), u.email_notifications),
+        where: u.local,
         where: not is_nil(u.email),
         where: u.last_digest_emailed_at < datetime_add(^now, ^negative_interval, "day"),
         select: u

priv/repo/migrations/20251117000000_restrict_eligible_user_indexes.exs

@@ -0,0 +1,56 @@
+defmodule Pleroma.Repo.Migrations.RestrictEligibleUserIndexes do
+  use Ecto.Migration
+
+  @old_indexes [
+    index(:users, [:email], unique: true),
+    index(:users, [:is_admin]),
+    index(:users, [:is_moderator]),
+    index(:users, [:is_suggested]),
+    index(:users, [:last_active_at])
+  ]
+
+  @new_indexes [
+    # We use this to send out emails to local users, i.e. we only care about
+    # _local_ users who also set an email to begin with
+    # (and to ensure no two local users claim the same email address;
+    #  though if we somehow get an email for a remote user, we don't care about collisions)
+    index(:users, [:email], unique: true, where: "email IS NOT NULL AND local"),
+
+    # Just used to quickly retrieve all suggested useres
+    # (this perhaps should have been a separate table to begin with).
+    # This MUST use BTREE, a HASH index will not be used when querying all suggested users!
+    index(:users, [:id], where: "is_suggested", using: :btree, name: :users_where_suggested_index),
+
+    # Only _local_ users can be admins or moderators and in practice
+    # this criteria is only used to query for a "true" setting.
+    # According to EXPLAIN, restricting just by the "true" state even performs _slightly_ better
+    # and requires less to keep in mind when contructing queries
+    index(:users, [:is_admin], where: "is_admin"),
+    index(:users, [:is_moderator], where: "is_moderator"),
+
+    # The following is only set and used for _local_ users
+    index(:users, [:last_active_at], where: "local")
+  ]
+
+  defp drop_all(indexes) do
+    for idx <- indexes do
+      drop_if_exists(idx)
+    end
+  end
+
+  defp create_all(indexes) do
+    for idx <- indexes do
+      create_if_not_exists(idx)
+    end
+  end
+
+  def up() do
+    drop_all(@old_indexes)
+    create_all(@new_indexes)
+  end
+
+  def down() do
+    drop_all(@new_indexes)
+    create_all(@old_indexes)
+  end
+end

test/pleroma/web/mastodon_api/controllers/account_controller_test.exs

@@ -482,17 +482,18 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do
       assert [%{"id" => ^post_id}] = json_response_and_validate_schema(conn, 200)
     end
 
-    test "the user views their own timelines and excludes direct messages", %{
+    test "the user views their own timelines and exclude_visibilites will be ignored", %{
       user: user,
       conn: conn
     } do
-      {:ok, %{id: public_activity_id}} =
+      {:ok, %{id: _public_activity_id}} =
         CommonAPI.post(user, %{status: ".", visibility: "public"})
 
       {:ok, _direct_activity} = CommonAPI.post(user, %{status: ".", visibility: "direct"})
 
       conn = get(conn, "/api/v1/accounts/#{user.id}/statuses?exclude_visibilities[]=direct")
-      assert [%{"id" => ^public_activity_id}] = json_response_and_validate_schema(conn, 200)
+      # assert [%{"id" => ^public_activity_id}] = json_response_and_validate_schema(conn, 200)
+      %{"error" => "Unexpected field: exclude_visibilities."} = json_response(conn, 400)
     end
 
     test "muted reactions", %{user: user, conn: conn} do

test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs

@@ -192,6 +192,18 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do
   end
 
   describe "exclude_visibilities" do
+    test "will be ignored" do
+      %{conn: conn} = oauth_access(["read:notifications"])
+
+      resp =
+        conn
+        |> get("/api/v1/notifications?exclude_visibilities[]=unlisted")
+        |> json_response(400)
+
+      %{"error" => "Unexpected field: exclude_visibilities."} = resp
+    end
+
+    @tag :skip
     test "filters notifications for mentions" do
       %{user: user, conn: conn} = oauth_access(["read:notifications"])
       other_user = insert(:user)
@@ -233,6 +245,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do
       assert id == public_activity.id
     end
 
+    @tag :skip
     test "filters notifications for Like activities" do
       user = insert(:user)
       %{user: other_user, conn: conn} = oauth_access(["read:notifications"])
@@ -300,6 +313,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do
       assert direct_activity.id in activity_ids
     end
 
+    @tag :skip
     test "filters notifications for Announce activities" do
       user = insert(:user)
       %{user: other_user, conn: conn} = oauth_access(["read:notifications"])
@@ -322,6 +336,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do
       refute unlisted_activity.id in activity_ids
     end
 
+    @tag :skip
     test "doesn't return less than the requested amount of records when the user's reply is liked" do
       user = insert(:user)
       %{user: other_user, conn: conn} = oauth_access(["read:notifications"])

test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs

@@ -38,21 +38,23 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do
              end)
     end
 
-    test "the home timeline when the direct messages are excluded", %{user: user, conn: conn} do
-      {:ok, public_activity} = CommonAPI.post(user, %{status: ".", visibility: "public"})
-      {:ok, direct_activity} = CommonAPI.post(user, %{status: ".", visibility: "direct"})
+    test "the home timeline ignores exclude_visibilities", %{user: user, conn: conn} do
+      {:ok, _public_activity} = CommonAPI.post(user, %{status: ".", visibility: "public"})
+      {:ok, _direct_activity} = CommonAPI.post(user, %{status: ".", visibility: "direct"})
 
-      {:ok, unlisted_activity} = CommonAPI.post(user, %{status: ".", visibility: "unlisted"})
+      {:ok, _unlisted_activity} = CommonAPI.post(user, %{status: ".", visibility: "unlisted"})
 
-      {:ok, private_activity} = CommonAPI.post(user, %{status: ".", visibility: "private"})
+      {:ok, _private_activity} = CommonAPI.post(user, %{status: ".", visibility: "private"})
 
       conn = get(conn, "/api/v1/timelines/home?exclude_visibilities[]=direct")
 
-      assert status_ids = json_response_and_validate_schema(conn, :ok) |> Enum.map(& &1["id"])
-      assert public_activity.id in status_ids
-      assert unlisted_activity.id in status_ids
-      assert private_activity.id in status_ids
-      refute direct_activity.id in status_ids
+      %{"error" => "Unexpected field: exclude_visibilities."} = json_response(conn, 400)
+
+      # assert status_ids = json_response_and_validate_schema(conn, :ok) |> Enum.map(& &1["id"])
+      # assert public_activity.id in status_ids
+      # assert unlisted_activity.id in status_ids
+      # assert private_activity.id in status_ids
+      # refute direct_activity.id in status_ids
     end
 
     test "muted emotions", %{user: user, conn: conn} do