From aac6086ca62c66b25f09bc8f33943dad7242d5ce Mon Sep 17 00:00:00 2001 From: Oneric Date: Tue, 11 Nov 2025 00:00:00 +0000 Subject: [PATCH 1/3] db: convert indexes to partial ones where appropriate This reduces both the size the indexes take on the disk and the overhead of maintaining it since it now only needs to be updated for some entries. For the email index restriction queries needed to be updated, the last_active_at index is used only by one query which already also restricted by "local" and for the remaining restrictions no query adaptations are needed. --- .../tasks/pleroma/notification_settings.ex | 1 + .../workers/cron/digest_emails_worker.ex | 1 + ...7000000_restrict_eligible_user_indexes.exs | 56 +++++++++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 priv/repo/migrations/20251117000000_restrict_eligible_user_indexes.exs diff --git a/lib/mix/tasks/pleroma/notification_settings.ex b/lib/mix/tasks/pleroma/notification_settings.ex index e16866b6a..696376a55 100644 --- a/lib/mix/tasks/pleroma/notification_settings.ex +++ b/lib/mix/tasks/pleroma/notification_settings.ex @@ -43,6 +43,7 @@ defmodule Mix.Tasks.Pleroma.NotificationSettings do defp build_query(hide_notification_contents, options) do query = from(u in Pleroma.User, + where: u.local, update: [ set: [ notification_settings: diff --git a/lib/pleroma/workers/cron/digest_emails_worker.ex b/lib/pleroma/workers/cron/digest_emails_worker.ex index 83dc75d60..22d53b227 100644 --- a/lib/pleroma/workers/cron/digest_emails_worker.ex +++ b/lib/pleroma/workers/cron/digest_emails_worker.ex @@ -31,6 +31,7 @@ defmodule Pleroma.Workers.Cron.DigestEmailsWorker do from(u in inactive_users_query, where: fragment(~s(? ->'digest' @> 'true'), u.email_notifications), + where: u.local, where: not is_nil(u.email), where: u.last_digest_emailed_at < datetime_add(^now, ^negative_interval, "day"), select: u diff --git a/priv/repo/migrations/20251117000000_restrict_eligible_user_indexes.exs b/priv/repo/migrations/20251117000000_restrict_eligible_user_indexes.exs new file mode 100644 index 000000000..7503a9157 --- /dev/null +++ b/priv/repo/migrations/20251117000000_restrict_eligible_user_indexes.exs @@ -0,0 +1,56 @@ +defmodule Pleroma.Repo.Migrations.RestrictEligibleUserIndexes do + use Ecto.Migration + + @old_indexes [ + index(:users, [:email], unique: true), + index(:users, [:is_admin]), + index(:users, [:is_moderator]), + index(:users, [:is_suggested]), + index(:users, [:last_active_at]) + ] + + @new_indexes [ + # We use this to send out emails to local users, i.e. we only care about + # _local_ users who also set an email to begin with + # (and to ensure no two local users claim the same email address; + # though if we somehow get an email for a remote user, we don't care about collisions) + index(:users, [:email], unique: true, where: "email IS NOT NULL AND local"), + + # Just used to quickly retrieve all suggested useres + # (this perhaps should have been a separate table to begin with). + # This MUST use BTREE, a HASH index will not be used when querying all suggested users! + index(:users, [:id], where: "is_suggested", using: :btree, name: :users_where_suggested_index), + + # Only _local_ users can be admins or moderators and in practice + # this criteria is only used to query for a "true" setting. + # According to EXPLAIN, restricting just by the "true" state even performs _slightly_ better + # and requires less to keep in mind when contructing queries + index(:users, [:is_admin], where: "is_admin"), + index(:users, [:is_moderator], where: "is_moderator"), + + # The following is only set and used for _local_ users + index(:users, [:last_active_at], where: "local") + ] + + defp drop_all(indexes) do + for idx <- indexes do + drop_if_exists(idx) + end + end + + defp create_all(indexes) do + for idx <- indexes do + create_if_not_exists(idx) + end + end + + def up() do + drop_all(@old_indexes) + create_all(@new_indexes) + end + + def down() do + drop_all(@new_indexes) + create_all(@old_indexes) + end +end From 0aeeaeb973c09c3533dd04ef9098bad1ae2a2451 Mon Sep 17 00:00:00 2001 From: Oneric Date: Thu, 27 Nov 2025 00:00:00 +0000 Subject: [PATCH 2/3] api: tentatively remove exclude_visibility parameter This was originally added in a97b642289659a5fccb5943c54caa1ecdce5fd2f as part of https://git.pleroma.social/pleroma/pleroma/-/merge_requests/1818 with the hope/intent of using it to hide DMs by default from timelines. This never came to be with thisd parameter still being unused today in al of akkoma-fe, pleroma-fe, Husky, pl-fe, Mangange, fedibird-fe and masto-fe. Given thypically only a small percentage of posts are DMs, as well as issues with the SQL function and index used to implement this a removal likely won't bother anyone while allowing us to clean up bugged code and improve actually used functions. While unlikely, we don't have a way to ascertain for sure whether there are still users so for now only drop it from API Spec which leads to the parameter being stripped out before processing and keep the actual logic. If any users exist and complain we can easily revert this, if not we will follow up with a proper cleanup. --- .../api_spec/operations/account_operation.ex | 6 ----- .../operations/notification_operation.ex | 7 ------ .../api_spec/operations/timeline_operation.ex | 20 +++-------------- lib/pleroma/web/mastodon_api/mastodon_api.ex | 2 +- .../controllers/account_controller_test.exs | 7 +++--- .../notification_controller_test.exs | 15 +++++++++++++ .../controllers/timeline_controller_test.exs | 22 ++++++++++--------- 7 files changed, 35 insertions(+), 44 deletions(-) diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index 11dce35c9..e02ad4271 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -138,12 +138,6 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do ), Operation.parameter(:exclude_reblogs, :query, BooleanLike, "Exclude reblogs"), Operation.parameter(:exclude_replies, :query, BooleanLike, "Exclude replies"), - Operation.parameter( - :exclude_visibilities, - :query, - %Schema{type: :array, items: VisibilityScope}, - "Exclude visibilities" - ), Operation.parameter( :with_muted, :query, diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex index f96ad70ef..f4e9a7706 100644 --- a/lib/pleroma/web/api_spec/operations/notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex @@ -10,7 +10,6 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do alias Pleroma.Web.ApiSpec.Schemas.ApiError alias Pleroma.Web.ApiSpec.Schemas.BooleanLike alias Pleroma.Web.ApiSpec.Schemas.Status - alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope import Pleroma.Web.ApiSpec.Helpers @@ -41,12 +40,6 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do %Schema{type: :string}, "Return only notifications received from this account" ), - Operation.parameter( - :exclude_visibilities, - :query, - %Schema{type: :array, items: VisibilityScope}, - "Exclude the notifications for activities with the given visibilities" - ), Operation.parameter( :include_types, :query, diff --git a/lib/pleroma/web/api_spec/operations/timeline_operation.ex b/lib/pleroma/web/api_spec/operations/timeline_operation.ex index 45c97cab6..5a91a7209 100644 --- a/lib/pleroma/web/api_spec/operations/timeline_operation.ex +++ b/lib/pleroma/web/api_spec/operations/timeline_operation.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.ApiSpec.TimelineOperation do alias Pleroma.Web.ApiSpec.Schemas.ApiError alias Pleroma.Web.ApiSpec.Schemas.BooleanLike alias Pleroma.Web.ApiSpec.Schemas.Status - alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope import Pleroma.Web.ApiSpec.Helpers @@ -28,7 +27,6 @@ defmodule Pleroma.Web.ApiSpec.TimelineOperation do remote_param(), only_media_param(), with_muted_param(), - exclude_visibilities_param(), reply_visibility_param() | pagination_params() ], operationId: "TimelineController.home", @@ -64,7 +62,6 @@ defmodule Pleroma.Web.ApiSpec.TimelineOperation do only_media_param(), remote_param(), with_muted_param(), - exclude_visibilities_param(), reply_visibility_param() | pagination_params() ], operationId: "TimelineController.public", @@ -85,7 +82,6 @@ defmodule Pleroma.Web.ApiSpec.TimelineOperation do only_media_param(), remote_param(), with_muted_param(), - exclude_visibilities_param(), reply_visibility_param() | pagination_params() ], operationId: "TimelineController.bubble", @@ -131,8 +127,7 @@ defmodule Pleroma.Web.ApiSpec.TimelineOperation do local_param(), only_media_param(), remote_param(), - with_muted_param(), - exclude_visibilities_param() | pagination_params() + with_muted_param() ], operationId: "TimelineController.hashtag", responses: %{ @@ -159,8 +154,8 @@ defmodule Pleroma.Web.ApiSpec.TimelineOperation do with_muted_param(), local_param(), remote_param(), - only_media_param(), - exclude_visibilities_param() | pagination_params() + only_media_param() + | pagination_params() ], operationId: "TimelineController.list", responses: %{ @@ -200,15 +195,6 @@ defmodule Pleroma.Web.ApiSpec.TimelineOperation do Operation.parameter(:with_muted, :query, BooleanLike, "Include activities by muted users") end - defp exclude_visibilities_param do - Operation.parameter( - :exclude_visibilities, - :query, - %Schema{type: :array, items: VisibilityScope}, - "Exclude the statuses with the given visibilities" - ) - end - defp reply_visibility_param do Operation.parameter( :reply_visibility, diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index 700474137..fd0106c4a 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -81,7 +81,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do param_types = %{ exclude_types: {:array, :string}, types: {:array, :string}, - exclude_visibilities: {:array, :string}, + # exclude_visibilities: {:array, :string}, reblogs: :boolean, with_muted: :boolean, account_ap_id: :string, diff --git a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs index ede14030b..19e656df0 100644 --- a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs @@ -482,17 +482,18 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do assert [%{"id" => ^post_id}] = json_response_and_validate_schema(conn, 200) end - test "the user views their own timelines and excludes direct messages", %{ + test "the user views their own timelines and exclude_visibilites will be ignored", %{ user: user, conn: conn } do - {:ok, %{id: public_activity_id}} = + {:ok, %{id: _public_activity_id}} = CommonAPI.post(user, %{status: ".", visibility: "public"}) {:ok, _direct_activity} = CommonAPI.post(user, %{status: ".", visibility: "direct"}) conn = get(conn, "/api/v1/accounts/#{user.id}/statuses?exclude_visibilities[]=direct") - assert [%{"id" => ^public_activity_id}] = json_response_and_validate_schema(conn, 200) + # assert [%{"id" => ^public_activity_id}] = json_response_and_validate_schema(conn, 200) + %{"error" => "Unexpected field: exclude_visibilities."} = json_response(conn, 400) end test "muted reactions", %{user: user, conn: conn} do diff --git a/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs index b761bff3f..06b0c9537 100644 --- a/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs @@ -192,6 +192,18 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do end describe "exclude_visibilities" do + test "will be ignored" do + %{conn: conn} = oauth_access(["read:notifications"]) + + resp = + conn + |> get("/api/v1/notifications?exclude_visibilities[]=unlisted") + |> json_response(400) + + %{"error" => "Unexpected field: exclude_visibilities."} = resp + end + + @tag :skip test "filters notifications for mentions" do %{user: user, conn: conn} = oauth_access(["read:notifications"]) other_user = insert(:user) @@ -233,6 +245,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do assert id == public_activity.id end + @tag :skip test "filters notifications for Like activities" do user = insert(:user) %{user: other_user, conn: conn} = oauth_access(["read:notifications"]) @@ -300,6 +313,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do assert direct_activity.id in activity_ids end + @tag :skip test "filters notifications for Announce activities" do user = insert(:user) %{user: other_user, conn: conn} = oauth_access(["read:notifications"]) @@ -322,6 +336,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do refute unlisted_activity.id in activity_ids end + @tag :skip test "doesn't return less than the requested amount of records when the user's reply is liked" do user = insert(:user) %{user: other_user, conn: conn} = oauth_access(["read:notifications"]) diff --git a/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs index ccaf17fe4..696438a79 100644 --- a/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs @@ -38,21 +38,23 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do end) end - test "the home timeline when the direct messages are excluded", %{user: user, conn: conn} do - {:ok, public_activity} = CommonAPI.post(user, %{status: ".", visibility: "public"}) - {:ok, direct_activity} = CommonAPI.post(user, %{status: ".", visibility: "direct"}) + test "the home timeline ignores exclude_visibilities", %{user: user, conn: conn} do + {:ok, _public_activity} = CommonAPI.post(user, %{status: ".", visibility: "public"}) + {:ok, _direct_activity} = CommonAPI.post(user, %{status: ".", visibility: "direct"}) - {:ok, unlisted_activity} = CommonAPI.post(user, %{status: ".", visibility: "unlisted"}) + {:ok, _unlisted_activity} = CommonAPI.post(user, %{status: ".", visibility: "unlisted"}) - {:ok, private_activity} = CommonAPI.post(user, %{status: ".", visibility: "private"}) + {:ok, _private_activity} = CommonAPI.post(user, %{status: ".", visibility: "private"}) conn = get(conn, "/api/v1/timelines/home?exclude_visibilities[]=direct") - assert status_ids = json_response_and_validate_schema(conn, :ok) |> Enum.map(& &1["id"]) - assert public_activity.id in status_ids - assert unlisted_activity.id in status_ids - assert private_activity.id in status_ids - refute direct_activity.id in status_ids + %{"error" => "Unexpected field: exclude_visibilities."} = json_response(conn, 400) + + # assert status_ids = json_response_and_validate_schema(conn, :ok) |> Enum.map(& &1["id"]) + # assert public_activity.id in status_ids + # assert unlisted_activity.id in status_ids + # assert private_activity.id in status_ids + # refute direct_activity.id in status_ids end test "muted emotions", %{user: user, conn: conn} do From 453ab11fb2eb7528678fbddf583a47dd2ae47d45 Mon Sep 17 00:00:00 2001 From: Oneric Date: Thu, 27 Nov 2025 00:00:00 +0000 Subject: [PATCH 3/3] changelog: add missing entries --- CHANGELOG.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2a1423bc..800e42a5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,16 +6,55 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased ### REMOVED +- DEPRECATE `config :pleroma, :instance, skip_thread_containment: false`. + It is due to be removed in one of the next releases if no strong arguments for keeping it are brought up. + It is already semi-broken for large threads and conflicts with pending optimisation and cleanup work. +- support for `exclude_visibilities` in timeline and notification endpoints has been dropped +- support for list visibility / list addressing has been dropped due to lack of usage, maintenance burden and redundancy with the still supported explicit-addressing feature +- support for conversations addressing has been dropped due to lack of usage, maintenance burden and being mostly redundant with explicit addressing +- per-visibility status counters have been dropped from `/api/v1/pleroma/admin/stats` + due to unreasonably perf costs added on most database operations. + For now, the response still contains the fields, but with stubbed-out values. ### Added - status responses include two new fields for ActivityPub cross-referencing: `akkoma.quote_apid` and `akkoma.in_reply_to_apid` +- attempting to reply to an already deleted post will return an error + (in akkoma-fe the error will be shown and your draft message retained so you can decide + for yourself whether to discard it or copy and repost as a, now intentional, new thread) +- the notification endpoint now supports the `types` parameter for filtering added in vanilla Mastodon +- the mute endpoint now supports the `duration` parameter added in vanilla Mastodon + (fixes temporary mutes created via e.g. Husky) ### Fixed - replies and quotes to unresolvable posts now fill out IDs for replied to status, user or quoted status with a 404-ing ID to make them recognisable as replies/quotes instead of pretending they’re root posts +- querying a status using the ID of a non-post AP activity no longer displays + a duplicate of the post referenced by said activity with mangled author information +- fix users being able to interact (like, emoji react, ...) with posts they cannot access +- fix AP fetches of local non-Create, non-Undo activities exposing the raw, unsanitised content of the referenced object +- the above two combined allowed local users to gain access to private posts + of user they do not follow, but follow a follower of the author. + (remote users and other scenarios were to our knowledge not able to achieve this due to other restrictions) +- fix RSS and Atom feeds of hashtag timelines potentially exposing more information than Mastodon API when restricting unauthenticated API access +- fix mentioning and sending DMs to users with non-ASCII-alphanumerical usernames +- correctly hide and show inlined fallback links for quotes from Mastodon instances +- API requests with multiple unsupported parameters now will ignore all of them up to a certain limit. + If there are too many unsupported parameters this is indicated in the returned error message. +- expose generic type of attachment via Masto API if remote did not send a full MIME type but indicated a generic one + (the \*oma-specific full mime type field in the API response remains generic however, since we don't have this info) +- add back the default banner image we advertise in Masto API +- correctly redirect `/users/:nickname.rss` to the RSS instead of Atom feed ### Changed +- depreacted the `included_types` parameter in the notification endpoint; replaced by `types` +- depreacted the `expires_in` parameter in the mute endpoint; replaced by `duration` +- optimised emoji addition and removal +- emoji reloading now happens asynchronously so you won't run into timeout issues with many emoji and/or a slow disk +- upgraded all of our dependencies; this should reduce issues when running akoma with OTP28 +- prefer "summary" over "name" for the attachment alt text of incoming ActivityPub documents; + this fixes alt text federation from GtS and Honk +- slightly improve index overhead for the users table ## 2025.10