From d2364276a1cbfb2b6b3851fd9fb0e48d773b923b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 10 Oct 2020 01:21:57 -0500 Subject: [PATCH 01/25] Blocks: always see your own posts --- lib/pleroma/web/activity_pub/activity_pub.ex | 23 +++++++++++++---- test/web/activity_pub/activity_pub_test.exs | 27 ++++++++++++++++++++ 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index eb44cffec..bf89c228a 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -791,10 +791,10 @@ defp restrict_replies(query, %{ where: fragment( """ - ?->>'type' != 'Create' -- This isn't a Create + ?->>'type' != 'Create' -- This isn't a Create OR ?->>'inReplyTo' is null -- this isn't a reply - OR ? && array_remove(?, ?) -- The recipient is us or one of our friends, - -- unless they are the author (because authors + OR ? && array_remove(?, ?) -- The recipient is us or one of our friends, + -- unless they are the author (because authors -- are also part of the recipients). This leads -- to a bug that self-replies by friends won't -- show up. @@ -850,7 +850,10 @@ defp restrict_blocked(query, %{blocking_user: %User{} = user} = opts) do from( [activity, object: o] in query, + # You don't block the author where: fragment("not (? = ANY(?))", activity.actor, ^blocked_ap_ids), + + # You don't block any recipients, and didn't author the post where: fragment( "((not (? && ?)) or ? = ?)", @@ -859,12 +862,18 @@ defp restrict_blocked(query, %{blocking_user: %User{} = user} = opts) do activity.actor, ^user.ap_id ), + + # You don't block the domain of any recipients, and didn't author the post where: fragment( - "recipients_contain_blocked_domains(?, ?) = false", + "(recipients_contain_blocked_domains(?, ?) = false) or ? = ?", activity.recipients, - ^domain_blocks + ^domain_blocks, + activity.actor, + ^user.ap_id ), + + # It's not a boost of a user you block where: fragment( "not (?->>'type' = 'Announce' and ?->'to' \\?| ?)", @@ -872,6 +881,8 @@ defp restrict_blocked(query, %{blocking_user: %User{} = user} = opts) do activity.data, ^blocked_ap_ids ), + + # You don't block the author's domain, and also don't follow the author where: fragment( "(not (split_part(?, '/', 3) = ANY(?))) or ? = ANY(?)", @@ -880,6 +891,8 @@ defp restrict_blocked(query, %{blocking_user: %User{} = user} = opts) do activity.actor, ^following_ap_ids ), + + # Same as above, but checks the Object where: fragment( "(not (split_part(?->>'actor', '/', 3) = ANY(?))) or (?->>'actor') = ANY(?)", diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 804305a13..e4661b478 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -622,6 +622,18 @@ test "doesn't return blocked activities" do assert Enum.member?(activities, activity_one) end + test "always see your own posts even when they address people you block" do + user = insert(:user) + blockee = insert(:user) + + {:ok, _} = User.block(user, blockee) + {:ok, activity} = CommonAPI.post(user, %{status: "hey! @#{blockee.nickname}"}) + + activities = ActivityPub.fetch_activities([], %{blocking_user: user}) + + assert Enum.member?(activities, activity) + end + test "doesn't return transitive interactions concerning blocked users" do blocker = insert(:user) blockee = insert(:user) @@ -721,6 +733,21 @@ test "doesn't return activities from blocked domains" do refute repeat_activity in activities end + test "see your own posts even when they adress actors from blocked domains" do + user = insert(:user) + + domain = "dogwhistle.zone" + domain_user = insert(:user, %{ap_id: "https://#{domain}/@pundit"}) + + {:ok, user} = User.block_domain(user, domain) + + {:ok, activity} = CommonAPI.post(user, %{status: "hey! @#{domain_user.nickname}"}) + + activities = ActivityPub.fetch_activities([], %{blocking_user: user}) + + assert Enum.member?(activities, activity) + end + test "does return activities from followed users on blocked domains" do domain = "meanies.social" domain_user = insert(:user, %{ap_id: "https://#{domain}/@pundit"}) From 2fc7ce3e1e2fb746305944d40ac74da16d48f7aa Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 10 Oct 2020 01:29:41 -0500 Subject: [PATCH 02/25] Blocks: add blockers_visible config --- config/config.exs | 1 + config/description.exs | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/config/config.exs b/config/config.exs index d53663d36..befc6840d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -363,6 +363,7 @@ config :pleroma, :activitypub, unfollow_blocked: true, outgoing_blocks: true, + blockers_visible: true, follow_handshake_timeout: 500, note_replies_output_limit: 5, sign_object_fetches: true, diff --git a/config/description.exs b/config/description.exs index 3902b9632..4f495e828 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2092,6 +2092,11 @@ type: :boolean, description: "Whether to federate blocks to other instances" }, + %{ + key: :blockers_visible, + type: :boolean, + description: "Whether a user can see someone who has blocked them" + }, %{ key: :sign_object_fetches, type: :boolean, From 7c2d0e378c45dd1c03fbd4a9d338bbeb4f9d2610 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 10 Oct 2020 03:41:35 -0500 Subject: [PATCH 03/25] Blocks: make blockers_visible config work --- lib/pleroma/web/activity_pub/activity_pub.ex | 27 +++++++++++++++++++ .../controllers/timeline_controller_test.exs | 18 +++++++++++++ 2 files changed, 45 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index bf89c228a..c9712d245 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -410,6 +410,7 @@ def fetch_activities_for_context_query(context, opts) do |> maybe_preload_bookmarks(opts) |> maybe_set_thread_muted_field(opts) |> restrict_blocked(opts) + |> restrict_blockers_visibility(opts) |> restrict_recipients(recipients, opts[:user]) |> restrict_filtered(opts) |> where( @@ -906,6 +907,31 @@ defp restrict_blocked(query, %{blocking_user: %User{} = user} = opts) do defp restrict_blocked(query, _), do: query + defp restrict_blockers_visibility(query, %{blocking_user: %User{} = user}) do + if Config.get([:activitypub, :blockers_visible]) == true do + query + else + blocker_ap_ids = User.incoming_relationships_ungrouped_ap_ids(user, [:block]) + + from( + activity in query, + # The author doesn't block you + where: fragment("not (? = ANY(?))", activity.actor, ^blocker_ap_ids), + + # It's not a boost of a user that blocks you + where: + fragment( + "not (?->>'type' = 'Announce' and ?->'to' \\?| ?)", + activity.data, + activity.data, + ^blocker_ap_ids + ) + ) + end + end + + defp restrict_blockers_visibility(query, _), do: query + defp restrict_unlisted(query, %{restrict_unlisted: true}) do from( activity in query, @@ -1106,6 +1132,7 @@ def fetch_activities_query(recipients, opts \\ %{}) do |> restrict_state(opts) |> restrict_favorited_by(opts) |> restrict_blocked(restrict_blocked_opts) + |> restrict_blockers_visibility(opts) |> restrict_muted(restrict_muted_opts) |> restrict_filtered(opts) |> restrict_media(opts) diff --git a/test/web/mastodon_api/controllers/timeline_controller_test.exs b/test/web/mastodon_api/controllers/timeline_controller_test.exs index c6e0268fd..e2a830811 100644 --- a/test/web/mastodon_api/controllers/timeline_controller_test.exs +++ b/test/web/mastodon_api/controllers/timeline_controller_test.exs @@ -126,6 +126,24 @@ test "doesn't return replies if follower is posting with blocked user" do [%{"id" => ^reply_from_me}, %{"id" => ^activity_id}] = response end + test "doesn't return posts from users who blocked you when :blockers_visible is disabled" do + clear_config([:activitypub, :blockers_visible], false) + + %{conn: conn, user: blockee} = oauth_access(["read:statuses"]) + blocker = insert(:user) + {:ok, _} = User.block(blocker, blockee) + + conn = assign(conn, :user, blockee) + + {:ok, _} = CommonAPI.post(blocker, %{status: "hey!"}) + + response = + get(conn, "/api/v1/timelines/public") + |> json_response_and_validate_schema(200) + + assert length(response) == 0 + end + test "doesn't return replies if follow is posting with users from blocked domain" do %{conn: conn, user: blocker} = oauth_access(["read:statuses"]) friend = insert(:user) From 5c8d2c468c3a14cde029e7aeedd0cdb580cce1df Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 10 Oct 2020 03:44:20 -0500 Subject: [PATCH 04/25] Blocks: update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fc1750d1..bb4c6d383 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`) - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`) - Mix task option for force-unfollowing relays +- `[:activitypub, :blockers_visible]` config to control visibility of blockers. ### Changed @@ -48,6 +49,7 @@ switched to a new configuration mechanism, however it was not officially removed - Add documented-but-missing chat pagination. - Allow sending out emails again. +- See your own post when addressing a user from a blocked domain. ## Unreleased (Patch) From 2aeb229de3f59e1704c633c31415e7e834f6ba67 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 4 Nov 2020 16:23:24 +0100 Subject: [PATCH 05/25] Cheatsheet: Add info about :blockers_visible --- docs/configuration/cheatsheet.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index ebf95ebc9..75cb4b348 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -205,6 +205,7 @@ config :pleroma, :mrf_user_allowlist, %{ ### :activitypub * `unfollow_blocked`: Whether blocks result in people getting unfollowed * `outgoing_blocks`: Whether to federate blocks to other instances +* `blockers_visible`: Whether a user can see the posts of users who blocked them * `deny_follow_blocked`: Whether to disallow following an account that has blocked the user in question * `sign_object_fetches`: Sign object fetches with HTTP signatures * `authorized_fetch_mode`: Require HTTP signatures for AP fetches From cc09079aea54f5ff754925e0d5fbbc3b268dcb6d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 6 Jan 2021 15:39:14 -0600 Subject: [PATCH 06/25] Exclude blockers from notifications when `blockers_visible: false` --- lib/pleroma/notification.ex | 12 ++++++++++++ .../notification_controller_test.exs | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index dd7a1c824..e9c9b3ea2 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -132,6 +132,7 @@ def for_user_query(user, opts \\ %{}) do |> preload([n, a, o], activity: {a, object: o}) |> exclude_notification_muted(user, exclude_notification_muted_opts) |> exclude_blocked(user, exclude_blocked_opts) + |> exclude_blockers(user) |> exclude_filtered(user) |> exclude_visibility(opts) end @@ -145,6 +146,17 @@ defp exclude_blocked(query, user, opts) do |> FollowingRelationship.keep_following_or_not_domain_blocked(user) end + defp exclude_blockers(query, user) do + if Pleroma.Config.get([:activitypub, :blockers_visible]) == true do + query + else + blocker_ap_ids = User.incoming_relationships_ungrouped_ap_ids(user, [:block]) + + query + |> where([n, a], a.actor not in ^blocker_ap_ids) + end + end + defp exclude_notification_muted(query, _, %{@include_muted_option => true}) do query end diff --git a/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs index 9ac8488f6..ef35bdd00 100644 --- a/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs @@ -103,6 +103,25 @@ test "by default, does not contain pleroma:report" do assert [_] = result end + test "excludes mentions from blockers when blockers_visible is false" do + clear_config([:activitypub, :blockers_visible], false) + + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + blocker = insert(:user) + + {:ok, _} = CommonAPI.block(blocker, user) + {:ok, activity} = CommonAPI.post(blocker, %{status: "hi @#{user.nickname}"}) + + {:ok, [_notification]} = Notification.create_notifications(activity) + + conn = + conn + |> assign(:user, user) + |> get("/api/v1/notifications") + + assert [] == json_response_and_validate_schema(conn, 200) + end + test "getting a single notification" do %{user: user, conn: conn} = oauth_access(["read:notifications"]) other_user = insert(:user) From b7b05a074867c1444dd539d6d2331f6d5504f6e6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 6 Jan 2021 15:42:36 -0600 Subject: [PATCH 07/25] Oopsie whoopsie fix changelog --- CHANGELOG.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10968d02b..dc1b4b6c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -141,13 +141,7 @@ switched to a new configuration mechanism, however it was not officially removed - Allow sending out emails again. - Allow sending chat messages to yourself - OStatus / static FE endpoints: fixed inaccessibility for anonymous users on non-federating instances, switched to handling per `:restrict_unauthenticated` setting. -<<<<<<< HEAD -- Mastodon API: Current user is now included in conversation if it's the only participant -- Mastodon API: Fixed last_status.account being not filled with account data -- See your own post when addressing a user from a blocked domain. -======= - Fix remote users with a whitespace name. ->>>>>>> upstream/develop ### Upgrade notes From 9a0cb34c8c3a3c47ad5074bd73ad5369ee852192 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 12 Jul 2021 16:20:15 -0500 Subject: [PATCH 08/25] Fix Activity.delete_all_by_object_ap_id/1 timeout so users can be deleted --- lib/pleroma/activity.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 6542e684e..0d2cb91cd 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -286,7 +286,7 @@ def delete_all_by_object_ap_id(id) when is_binary(id) do |> Queries.by_object_id() |> Queries.exclude_type("Delete") |> select([u], u) - |> Repo.delete_all() + |> Repo.delete_all(timeout: :infinity) |> elem(1) |> Enum.find(fn %{data: %{"type" => "Create", "object" => ap_id}} when is_binary(ap_id) -> ap_id == id From 65514484cc5d8457a85e6c9d162da20f32210489 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 12 Jul 2021 16:59:54 -0500 Subject: [PATCH 09/25] CHANGELOG: fixed user deletion database timeout --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 036b9e775..8befe7476 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Checking activated Upload Filters for required commands. - Remote users can no longer reappear after being deleted. - Deactivated users may now be deleted. +- Deleting an activity with a lot of likes/boosts no longer causes a database timeout. - Mix task `pleroma.database prune_objects` - Linkify: Parsing crash with URLs ending in unbalanced closed paren, no path separator, and no query parameters From 25676c84be5d7126f17e009ee87abd2df42fa65c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 17 Jul 2021 13:05:25 -0500 Subject: [PATCH 10/25] Create AdminAPI.InstanceController --- .../controllers/admin_api_controller.ex | 20 +----- .../controllers/instance_controller.ex | 50 +++++++++++++++ lib/pleroma/web/router.ex | 2 +- .../controllers/admin_api_controller_test.exs | 34 ---------- .../controllers/instance_controller_test.exs | 64 +++++++++++++++++++ 5 files changed, 116 insertions(+), 54 deletions(-) create mode 100644 lib/pleroma/web/admin_api/controllers/instance_controller.ex create mode 100644 test/pleroma/web/admin_api/controllers/instance_controller_test.exs diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index 839ac1a8d..50aa294f0 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -49,7 +49,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do plug( OAuthScopesPlug, %{scopes: ["admin:read:statuses"]} - when action in [:list_user_statuses, :list_instance_statuses] + when action in [:list_user_statuses] ) plug( @@ -81,24 +81,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do action_fallback(AdminAPI.FallbackController) - def list_instance_statuses(conn, %{"instance" => instance} = params) do - with_reblogs = params["with_reblogs"] == "true" || params["with_reblogs"] == true - {page, page_size} = page_params(params) - - result = - ActivityPub.fetch_statuses(nil, %{ - instance: instance, - limit: page_size, - offset: (page - 1) * page_size, - exclude_reblogs: not with_reblogs, - total: true - }) - - conn - |> put_view(AdminAPI.StatusView) - |> render("index.json", %{total: result[:total], activities: result[:items], as: :activity}) - end - def list_user_statuses(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname} = params) do with_reblogs = params["with_reblogs"] == "true" || params["with_reblogs"] == true godmode = params["godmode"] == "true" || params["godmode"] == true diff --git a/lib/pleroma/web/admin_api/controllers/instance_controller.ex b/lib/pleroma/web/admin_api/controllers/instance_controller.ex new file mode 100644 index 000000000..ad6ea89ef --- /dev/null +++ b/lib/pleroma/web/admin_api/controllers/instance_controller.ex @@ -0,0 +1,50 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.InstanceController do + use Pleroma.Web, :controller + + import Pleroma.Web.ControllerHelper, only: [fetch_integer_param: 3] + + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.AdminAPI + alias Pleroma.Web.Plugs.OAuthScopesPlug + + require Logger + + @default_page_size 50 + + plug( + OAuthScopesPlug, + %{scopes: ["admin:read:statuses"]} + when action in [:list_instance_statuses] + ) + + action_fallback(AdminAPI.FallbackController) + + def list_instance_statuses(conn, %{"instance" => instance} = params) do + with_reblogs = params["with_reblogs"] == "true" || params["with_reblogs"] == true + {page, page_size} = page_params(params) + + result = + ActivityPub.fetch_statuses(nil, %{ + instance: instance, + limit: page_size, + offset: (page - 1) * page_size, + exclude_reblogs: not with_reblogs, + total: true + }) + + conn + |> put_view(AdminAPI.StatusView) + |> render("index.json", %{total: result[:total], activities: result[:items], as: :activity}) + end + + defp page_params(params) do + { + fetch_integer_param(params, "page", 1), + fetch_integer_param(params, "page_size", @default_page_size) + } + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 72ad14f05..dbc7df104 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -209,7 +209,7 @@ defmodule Pleroma.Web.Router do get("/users/:nickname/statuses", AdminAPIController, :list_user_statuses) get("/users/:nickname/chats", AdminAPIController, :list_user_chats) - get("/instances/:instance/statuses", AdminAPIController, :list_instance_statuses) + get("/instances/:instance/statuses", InstanceController, :list_instance_statuses) get("/instance_document/:name", InstanceDocumentController, :show) patch("/instance_document/:name", InstanceDocumentController, :update) diff --git a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs index 8cd9f939b..a9a0c800b 100644 --- a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs @@ -800,40 +800,6 @@ test "sets password_reset_pending to true", %{conn: conn} do end end - describe "instances" do - test "GET /instances/:instance/statuses", %{conn: conn} do - user = insert(:user, local: false, ap_id: "https://archae.me/users/archaeme") - user2 = insert(:user, local: false, ap_id: "https://test.com/users/test") - insert_pair(:note_activity, user: user) - activity = insert(:note_activity, user: user2) - - %{"total" => 2, "activities" => activities} = - conn |> get("/api/pleroma/admin/instances/archae.me/statuses") |> json_response(200) - - assert length(activities) == 2 - - %{"total" => 1, "activities" => [_]} = - conn |> get("/api/pleroma/admin/instances/test.com/statuses") |> json_response(200) - - %{"total" => 0, "activities" => []} = - conn |> get("/api/pleroma/admin/instances/nonexistent.com/statuses") |> json_response(200) - - CommonAPI.repeat(activity.id, user) - - %{"total" => 2, "activities" => activities} = - conn |> get("/api/pleroma/admin/instances/archae.me/statuses") |> json_response(200) - - assert length(activities) == 2 - - %{"total" => 3, "activities" => activities} = - conn - |> get("/api/pleroma/admin/instances/archae.me/statuses?with_reblogs=true") - |> json_response(200) - - assert length(activities) == 3 - end - end - describe "PATCH /confirm_email" do test "it confirms emails of two users", %{conn: conn, admin: admin} do [first_user, second_user] = insert_pair(:user, is_confirmed: false) diff --git a/test/pleroma/web/admin_api/controllers/instance_controller_test.exs b/test/pleroma/web/admin_api/controllers/instance_controller_test.exs new file mode 100644 index 000000000..b70efcf23 --- /dev/null +++ b/test/pleroma/web/admin_api/controllers/instance_controller_test.exs @@ -0,0 +1,64 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.InstanceControllerTest do + use Pleroma.Web.ConnCase + use Oban.Testing, repo: Pleroma.Repo + + import Pleroma.Factory + + alias Pleroma.Web.CommonAPI + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + + :ok + end + + setup do + admin = insert(:user, is_admin: true) + token = insert(:oauth_admin_token, user: admin) + + conn = + build_conn() + |> assign(:user, admin) + |> assign(:token, token) + + {:ok, %{admin: admin, token: token, conn: conn}} + end + + describe "instances" do + test "GET /instances/:instance/statuses", %{conn: conn} do + user = insert(:user, local: false, ap_id: "https://archae.me/users/archaeme") + user2 = insert(:user, local: false, ap_id: "https://test.com/users/test") + insert_pair(:note_activity, user: user) + activity = insert(:note_activity, user: user2) + + %{"total" => 2, "activities" => activities} = + conn |> get("/api/pleroma/admin/instances/archae.me/statuses") |> json_response(200) + + assert length(activities) == 2 + + %{"total" => 1, "activities" => [_]} = + conn |> get("/api/pleroma/admin/instances/test.com/statuses") |> json_response(200) + + %{"total" => 0, "activities" => []} = + conn |> get("/api/pleroma/admin/instances/nonexistent.com/statuses") |> json_response(200) + + CommonAPI.repeat(activity.id, user) + + %{"total" => 2, "activities" => activities} = + conn |> get("/api/pleroma/admin/instances/archae.me/statuses") |> json_response(200) + + assert length(activities) == 2 + + %{"total" => 3, "activities" => activities} = + conn + |> get("/api/pleroma/admin/instances/archae.me/statuses?with_reblogs=true") + |> json_response(200) + + assert length(activities) == 3 + end + end +end From c136dc0987dda1b48298380b0db48bd92fef3f78 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 23 Jun 2021 13:02:41 -0500 Subject: [PATCH 11/25] Upgrade Ecto to v3.6.2, remove deprecated ecto_explain --- mix.exs | 2 +- mix.lock | 10 +++++----- test/mix/tasks/pleroma/ecto/migrate_test.exs | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mix.exs b/mix.exs index 436381f32..182b655ad 100644 --- a/mix.exs +++ b/mix.exs @@ -121,7 +121,7 @@ defp deps do {:phoenix_pubsub, "~> 2.0"}, {:phoenix_ecto, "~> 4.0"}, {:ecto_enum, "~> 1.4"}, - {:ecto_sql, "~> 3.4.4"}, + {:ecto_sql, "~> 3.6.2"}, {:postgrex, ">= 0.15.5"}, {:oban, "~> 2.3.4"}, {:gettext, "~> 0.18"}, diff --git a/mix.lock b/mix.lock index 99be81826..6e3b8b71c 100644 --- a/mix.lock +++ b/mix.lock @@ -24,14 +24,14 @@ "crontab": {:hex, :crontab, "1.1.8", "2ce0e74777dfcadb28a1debbea707e58b879e6aa0ffbf9c9bb540887bce43617", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "crypt": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/crypt.git", "cf2aa3f11632e8b0634810a15b3e612c7526f6a3", [ref: "cf2aa3f11632e8b0634810a15b3e612c7526f6a3"]}, "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"}, - "db_connection": {:hex, :db_connection, "2.3.1", "4c9f3ed1ef37471cbdd2762d6655be11e38193904d9c5c1c9389f1b891a3088e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm", "abaab61780dde30301d840417890bd9f74131041afd02174cf4e10635b3a63f5"}, + "db_connection": {:hex, :db_connection, "2.4.0", "d04b1b73795dae60cead94189f1b8a51cc9e1f911c234cc23074017c43c031e5", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ad416c21ad9f61b3103d254a71b63696ecadb6a917b36f563921e0de00d7d7c8"}, "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, "earmark": {:hex, :earmark, "1.4.3", "364ca2e9710f6bff494117dbbd53880d84bebb692dafc3a78eb50aa3183f2bfd", [:mix], [], "hexpm", "8cf8a291ebf1c7b9539e3cddb19e9cef066c2441b1640f13c34c1d3cfc825fec"}, "earmark_parser": {:hex, :earmark_parser, "1.4.10", "6603d7a603b9c18d3d20db69921527f82ef09990885ed7525003c7fe7dc86c56", [:mix], [], "hexpm", "8e2d5370b732385db2c9b22215c3f59c84ac7dda7ed7e544d7c459496ae519c0"}, - "ecto": {:hex, :ecto, "3.4.6", "08f7afad3257d6eb8613309af31037e16c36808dfda5a3cd0cb4e9738db030e4", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6f13a9e2a62e75c2dcfc7207bfc65645ab387af8360db4c89fee8b5a4bf3f70b"}, + "ecto": {:hex, :ecto, "3.6.2", "efdf52acfc4ce29249bab5417415bd50abd62db7b0603b8bab0d7b996548c2bc", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "efad6dfb04e6f986b8a3047822b0f826d9affe8e4ebdd2aeedbfcb14fd48884e"}, "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"}, - "ecto_sql": {:hex, :ecto_sql, "3.4.5", "30161f81b167d561a9a2df4329c10ae05ff36eca7ccc84628f2c8b9fa1e43323", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.0", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "31990c6a3579b36a3c0841d34a94c275e727de8b84f58509da5f1b2032c98ac2"}, + "ecto_sql": {:hex, :ecto_sql, "3.6.2", "9526b5f691701a5181427634c30655ac33d11e17e4069eff3ae1176c764e0ba3", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.6.2", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.4.0 or ~> 0.5.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5ec9d7e6f742ea39b63aceaea9ac1d1773d574ea40df5a53ef8afbd9242fdb6b"}, "eimp": {:hex, :eimp, "1.0.14", "fc297f0c7e2700457a95a60c7010a5f1dcb768a083b6d53f49cd94ab95a28f22", [:rebar3], [{:p1_utils, "1.0.18", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm", "501133f3112079b92d9e22da8b88bf4f0e13d4d67ae9c15c42c30bd25ceb83b6"}, "elixir_make": {:hex, :elixir_make, "0.6.2", "7dffacd77dec4c37b39af867cedaabb0b59f6a871f89722c25b28fcd4bd70530", [:mix], [], "hexpm", "03e49eadda22526a7e5279d53321d1cced6552f344ba4e03e619063de75348d9"}, "esshd": {:hex, :esshd, "0.1.1", "d4dd4c46698093a40a56afecce8a46e246eb35463c457c246dacba2e056f31b5", [:mix], [], "hexpm", "d73e341e3009d390aa36387dc8862860bf9f874c94d9fd92ade2926376f49981"}, @@ -97,7 +97,7 @@ "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "79fd4fcf34d110605c26560cbae8f23c603ec4158c08298bd4360fdea90bb5cf"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"}, - "postgrex": {:hex, :postgrex, "0.15.7", "724410acd48abac529d0faa6c2a379fb8ae2088e31247687b16cacc0e0883372", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "88310c010ff047cecd73d5ceca1d99205e4b1ab1b9abfdab7e00f5c9d20ef8f9"}, + "postgrex": {:hex, :postgrex, "0.15.9", "46f8fe6f25711aeb861c4d0ae09780facfdf3adbd2fb5594ead61504dd489bda", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "610719103e4cb2223d4ab78f9f0f3e720320eeca6011415ab4137ddef730adee"}, "pot": {:hex, :pot, "0.11.0", "61bad869a94534739dd4614a25a619bc5c47b9970e9a0ea5bef4628036fc7a16", [:rebar3], [], "hexpm", "57ee6ee6bdeb639661ffafb9acefe3c8f966e45394de6a766813bb9e1be4e54b"}, "prometheus": {:hex, :prometheus, "4.6.0", "20510f381db1ccab818b4cf2fac5fa6ab5cc91bc364a154399901c001465f46f", [:mix, :rebar3], [], "hexpm", "4905fd2992f8038eccd7aa0cd22f40637ed618c0bed1f75c05aacec15b7545de"}, "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.3", "3dd4da1812b8e0dbee81ea58bb3b62ed7588f2eae0c9e97e434c46807ff82311", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm", "8d66289f77f913b37eda81fd287340c17e61a447549deb28efc254532b2bed82"}, @@ -114,7 +114,7 @@ "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm", "2e1ec458f892ffa81f9f8386e3f35a1af6db7a7a37748a64478f13163a1f3573"}, "swoosh": {:hex, :swoosh, "1.0.6", "6765e334c67dacabe721f0d701c7e5a6f06e4595c90df6f91e73ebd54d555833", [:mix], [{:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "7c50ef78e4acfd1cbd4907dc1fa87b5540675a6be9dc979d04890f49d7ec1830"}, "syslog": {:hex, :syslog, "1.1.0", "6419a232bea84f07b56dc575225007ffe34d9fdc91abe6f1b2f254fd71d8efc2", [:rebar3], [], "hexpm", "4c6a41373c7e20587be33ef841d3de6f3beba08519809329ecc4d27b15b659e1"}, - "telemetry": {:hex, :telemetry, "0.4.2", "2808c992455e08d6177322f14d3bdb6b625fbcfd233a73505870d8738a2f4599", [:rebar3], [], "hexpm", "2d1419bd9dda6a206d7b5852179511722e2b18812310d304620c7bd92a13fcef"}, + "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, "tesla": {:hex, :tesla, "1.4.0", "1081bef0124b8bdec1c3d330bbe91956648fb008cf0d3950a369cda466a31a87", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.3", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "bf1374a5569f5fca8e641363b63f7347d680d91388880979a33bc12a6eb3e0aa"}, "timex": {:hex, :timex, "3.6.2", "845cdeb6119e2fef10751c0b247b6c59d86d78554c83f78db612e3290f819bc2", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "26030b46199d02a590be61c2394b37ea25a3664c02fafbeca0b24c972025d47a"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"}, diff --git a/test/mix/tasks/pleroma/ecto/migrate_test.exs b/test/mix/tasks/pleroma/ecto/migrate_test.exs index 5bdfd8f30..3bfdde1c0 100644 --- a/test/mix/tasks/pleroma/ecto/migrate_test.exs +++ b/test/mix/tasks/pleroma/ecto/migrate_test.exs @@ -13,7 +13,7 @@ test "ecto.migrate info message" do assert capture_log(fn -> Mix.Tasks.Pleroma.Ecto.Migrate.run() - end) =~ "[info] Already up" + end) =~ "[info] Migrations already up" Logger.configure(level: level) end From f67d00d12b5d66a960bd7b3822a5fa4418348e18 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 17 Jul 2021 14:06:35 -0500 Subject: [PATCH 12/25] Add Instance.delete_users_and_activities/1 to delete all content from a remote instance --- lib/pleroma/instances/instance.ex | 22 +++++++++++++++++ lib/pleroma/workers/background_worker.ex | 5 ++++ test/pleroma/instances/instance_test.exs | 31 ++++++++++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 4d0e8034d..2f338b3e2 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -8,6 +8,8 @@ defmodule Pleroma.Instances.Instance do alias Pleroma.Instances alias Pleroma.Instances.Instance alias Pleroma.Repo + alias Pleroma.User + alias Pleroma.Workers.BackgroundWorker use Ecto.Schema @@ -195,4 +197,24 @@ defp scrape_favicon(%URI{} = instance_uri) do nil end end + + @doc """ + Deletes all users from an instance in a background task, thus also deleting + all of those users' activities and notifications. + """ + def delete_users_and_activities(host) when is_binary(host) do + BackgroundWorker.enqueue("delete_instance", %{"host" => host}) + end + + def perform(:delete_instance, host) when is_binary(host) do + User.Query.build(%{nickname: "@#{host}"}) + |> Repo.chunk_stream(100, :batches) + |> Stream.each(fn users -> + users + |> Enum.each(fn user -> + User.perform(:delete, user) + end) + end) + |> Stream.run() + end end diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index 1e28384cb..4db077232 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.BackgroundWorker do + alias Pleroma.Instances.Instance alias Pleroma.User use Pleroma.Workers.WorkerHelper, queue: "background" @@ -38,4 +39,8 @@ def perform(%Job{ Pleroma.FollowingRelationship.move_following(origin, target) end + + def perform(%Job{args: %{"op" => "delete_instance", "host" => host}}) do + Instance.perform(:delete_instance, host) + end end diff --git a/test/pleroma/instances/instance_test.exs b/test/pleroma/instances/instance_test.exs index bacc0b19b..e49922724 100644 --- a/test/pleroma/instances/instance_test.exs +++ b/test/pleroma/instances/instance_test.exs @@ -6,6 +6,8 @@ defmodule Pleroma.Instances.InstanceTest do alias Pleroma.Instances alias Pleroma.Instances.Instance alias Pleroma.Repo + alias Pleroma.Tests.ObanHelpers + alias Pleroma.Web.CommonAPI use Pleroma.DataCase @@ -158,4 +160,33 @@ test "Doesn't scrapes unreachable instances" do "Instance.scrape_favicon(\"#{url}\") ignored unreachable host" end end + + test "delete_users_and_activities/1 deletes remote instance users and activities" do + [mario, luigi, _peach, wario] = + users = [ + insert(:user, nickname: "mario@mushroom.kingdom", name: "Mario"), + insert(:user, nickname: "luigi@mushroom.kingdom", name: "Luigi"), + insert(:user, nickname: "peach@mushroom.kingdom", name: "Peach"), + insert(:user, nickname: "wario@greedville.biz", name: "Wario") + ] + + {:ok, post1} = CommonAPI.post(mario, %{status: "letsa go!"}) + {:ok, post2} = CommonAPI.post(luigi, %{status: "itsa me... luigi"}) + {:ok, post3} = CommonAPI.post(wario, %{status: "WHA-HA-HA!"}) + + {:ok, job} = Instance.delete_users_and_activities("mushroom.kingdom") + :ok = ObanHelpers.perform(job) + + [mario, luigi, peach, wario] = Repo.reload(users) + + refute mario.is_active + refute luigi.is_active + refute peach.is_active + refute peach.name == "Peach" + + assert wario.is_active + assert wario.name == "Wario" + + assert [nil, nil, %{}] = Repo.reload([post1, post2, post3]) + end end From 54dbcfe02a05a75d7fe7591335df74743930182d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 17 Jul 2021 14:55:05 -0500 Subject: [PATCH 13/25] AdminAPI: add DELETE /instances/:instance to delete all content from a remote instance --- docs/development/API/admin_api.md | 16 ++++++++++++++++ .../controllers/instance_controller.ex | 13 +++++++++++++ lib/pleroma/web/router.ex | 1 + .../controllers/instance_controller_test.exs | 18 ++++++++++++++++++ 4 files changed, 48 insertions(+) diff --git a/docs/development/API/admin_api.md b/docs/development/API/admin_api.md index 8f855d251..82483fae7 100644 --- a/docs/development/API/admin_api.md +++ b/docs/development/API/admin_api.md @@ -319,6 +319,22 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret } ``` +## `DELETE /api/v1/pleroma/admin/instances/:instance` + +### Delete all users and activities from a remote instance + +Note: this will trigger a job to remove instance content in the background. +It may take some time. + +- Params: + - `instance`: remote instance host +- Response: + - The `instance` name as a string + +```json +"lain.com" +``` + ## `GET /api/v1/pleroma/admin/statuses` ### Retrives all latest statuses diff --git a/lib/pleroma/web/admin_api/controllers/instance_controller.ex b/lib/pleroma/web/admin_api/controllers/instance_controller.ex index ad6ea89ef..2fc2fb442 100644 --- a/lib/pleroma/web/admin_api/controllers/instance_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/instance_controller.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.AdminAPI.InstanceController do import Pleroma.Web.ControllerHelper, only: [fetch_integer_param: 3] + alias Pleroma.Instances.Instance alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.AdminAPI alias Pleroma.Web.Plugs.OAuthScopesPlug @@ -21,6 +22,12 @@ defmodule Pleroma.Web.AdminAPI.InstanceController do when action in [:list_instance_statuses] ) + plug( + OAuthScopesPlug, + %{scopes: ["admin:write:accounts", "admin:write:statuses"]} + when action in [:delete_instance] + ) + action_fallback(AdminAPI.FallbackController) def list_instance_statuses(conn, %{"instance" => instance} = params) do @@ -41,6 +48,12 @@ def list_instance_statuses(conn, %{"instance" => instance} = params) do |> render("index.json", %{total: result[:total], activities: result[:items], as: :activity}) end + def delete_instance(conn, %{"instance" => instance}) do + with {:ok, _job} <- Instance.delete_users_and_activities(instance) do + json(conn, instance) + end + end + defp page_params(params) do { fetch_integer_param(params, "page", 1), diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index dbc7df104..1f465b697 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -210,6 +210,7 @@ defmodule Pleroma.Web.Router do get("/users/:nickname/chats", AdminAPIController, :list_user_chats) get("/instances/:instance/statuses", InstanceController, :list_instance_statuses) + delete("/instances/:instance", InstanceController, :delete_instance) get("/instance_document/:name", InstanceDocumentController, :show) patch("/instance_document/:name", InstanceDocumentController, :update) diff --git a/test/pleroma/web/admin_api/controllers/instance_controller_test.exs b/test/pleroma/web/admin_api/controllers/instance_controller_test.exs index b70efcf23..42248c6bd 100644 --- a/test/pleroma/web/admin_api/controllers/instance_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/instance_controller_test.exs @@ -8,6 +8,8 @@ defmodule Pleroma.Web.AdminAPI.InstanceControllerTest do import Pleroma.Factory + alias Pleroma.Repo + alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.CommonAPI setup_all do @@ -61,4 +63,20 @@ test "GET /instances/:instance/statuses", %{conn: conn} do assert length(activities) == 3 end end + + test "DELETE /instances/:instance", %{conn: conn} do + user = insert(:user, nickname: "lain@lain.com") + post = insert(:note_activity, user: user) + + response = + conn + |> delete("/api/pleroma/admin/instances/lain.com") + |> json_response(200) + + [:ok] = ObanHelpers.perform_all() + + assert response == "lain.com" + refute Repo.reload(user).is_active + refute Repo.reload(post) + end end From bad79f79e65dc9b093f364393b802c67d632a5c6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 17 Jul 2021 15:00:15 -0500 Subject: [PATCH 14/25] AdminAPI.InstanceController: clean up tests, rename actions --- .../controllers/instance_controller.ex | 8 ++-- lib/pleroma/web/router.ex | 4 +- .../controllers/instance_controller_test.exs | 44 +++++++++---------- 3 files changed, 27 insertions(+), 29 deletions(-) diff --git a/lib/pleroma/web/admin_api/controllers/instance_controller.ex b/lib/pleroma/web/admin_api/controllers/instance_controller.ex index 2fc2fb442..00857983f 100644 --- a/lib/pleroma/web/admin_api/controllers/instance_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/instance_controller.ex @@ -19,18 +19,18 @@ defmodule Pleroma.Web.AdminAPI.InstanceController do plug( OAuthScopesPlug, %{scopes: ["admin:read:statuses"]} - when action in [:list_instance_statuses] + when action in [:list_statuses] ) plug( OAuthScopesPlug, %{scopes: ["admin:write:accounts", "admin:write:statuses"]} - when action in [:delete_instance] + when action in [:delete] ) action_fallback(AdminAPI.FallbackController) - def list_instance_statuses(conn, %{"instance" => instance} = params) do + def list_statuses(conn, %{"instance" => instance} = params) do with_reblogs = params["with_reblogs"] == "true" || params["with_reblogs"] == true {page, page_size} = page_params(params) @@ -48,7 +48,7 @@ def list_instance_statuses(conn, %{"instance" => instance} = params) do |> render("index.json", %{total: result[:total], activities: result[:items], as: :activity}) end - def delete_instance(conn, %{"instance" => instance}) do + def delete(conn, %{"instance" => instance}) do with {:ok, _job} <- Instance.delete_users_and_activities(instance) do json(conn, instance) end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 1f465b697..3e2ecc2c0 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -209,8 +209,8 @@ defmodule Pleroma.Web.Router do get("/users/:nickname/statuses", AdminAPIController, :list_user_statuses) get("/users/:nickname/chats", AdminAPIController, :list_user_chats) - get("/instances/:instance/statuses", InstanceController, :list_instance_statuses) - delete("/instances/:instance", InstanceController, :delete_instance) + get("/instances/:instance/statuses", InstanceController, :list_statuses) + delete("/instances/:instance", InstanceController, :delete) get("/instance_document/:name", InstanceDocumentController, :show) patch("/instance_document/:name", InstanceDocumentController, :update) diff --git a/test/pleroma/web/admin_api/controllers/instance_controller_test.exs b/test/pleroma/web/admin_api/controllers/instance_controller_test.exs index 42248c6bd..c78307fc8 100644 --- a/test/pleroma/web/admin_api/controllers/instance_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/instance_controller_test.exs @@ -30,38 +30,36 @@ defmodule Pleroma.Web.AdminAPI.InstanceControllerTest do {:ok, %{admin: admin, token: token, conn: conn}} end - describe "instances" do - test "GET /instances/:instance/statuses", %{conn: conn} do - user = insert(:user, local: false, ap_id: "https://archae.me/users/archaeme") - user2 = insert(:user, local: false, ap_id: "https://test.com/users/test") - insert_pair(:note_activity, user: user) - activity = insert(:note_activity, user: user2) + test "GET /instances/:instance/statuses", %{conn: conn} do + user = insert(:user, local: false, ap_id: "https://archae.me/users/archaeme") + user2 = insert(:user, local: false, ap_id: "https://test.com/users/test") + insert_pair(:note_activity, user: user) + activity = insert(:note_activity, user: user2) - %{"total" => 2, "activities" => activities} = - conn |> get("/api/pleroma/admin/instances/archae.me/statuses") |> json_response(200) + %{"total" => 2, "activities" => activities} = + conn |> get("/api/pleroma/admin/instances/archae.me/statuses") |> json_response(200) - assert length(activities) == 2 + assert length(activities) == 2 - %{"total" => 1, "activities" => [_]} = - conn |> get("/api/pleroma/admin/instances/test.com/statuses") |> json_response(200) + %{"total" => 1, "activities" => [_]} = + conn |> get("/api/pleroma/admin/instances/test.com/statuses") |> json_response(200) - %{"total" => 0, "activities" => []} = - conn |> get("/api/pleroma/admin/instances/nonexistent.com/statuses") |> json_response(200) + %{"total" => 0, "activities" => []} = + conn |> get("/api/pleroma/admin/instances/nonexistent.com/statuses") |> json_response(200) - CommonAPI.repeat(activity.id, user) + CommonAPI.repeat(activity.id, user) - %{"total" => 2, "activities" => activities} = - conn |> get("/api/pleroma/admin/instances/archae.me/statuses") |> json_response(200) + %{"total" => 2, "activities" => activities} = + conn |> get("/api/pleroma/admin/instances/archae.me/statuses") |> json_response(200) - assert length(activities) == 2 + assert length(activities) == 2 - %{"total" => 3, "activities" => activities} = - conn - |> get("/api/pleroma/admin/instances/archae.me/statuses?with_reblogs=true") - |> json_response(200) + %{"total" => 3, "activities" => activities} = + conn + |> get("/api/pleroma/admin/instances/archae.me/statuses?with_reblogs=true") + |> json_response(200) - assert length(activities) == 3 - end + assert length(activities) == 3 end test "DELETE /instances/:instance", %{conn: conn} do From c4efe0d2d62763b1163d734c20f19876b1b66865 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 17 Jul 2021 15:05:20 -0500 Subject: [PATCH 15/25] CHANGELOG: instance deletion --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 155248a81..d4c17c694 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - MRF (`FollowBotPolicy`): New MRF Policy which makes a designated local Bot account attempt to follow all users in public Notes received by your instance. Users who require approving follower requests or have #nobot in their profile are excluded. - Return OAuth token `id` (primary key) in POST `/oauth/token`. - AdminAPI: return `created_at` date with users. +- AdminAPI: add DELETE `/api/v1/pleroma/admin/instances/:instance` to delete all content from a remote instance. - `AnalyzeMetadata` upload filter for extracting image/video attachment dimensions and generating blurhashes for images. Blurhashes for videos are not generated at this time. - Attachment dimensions and blurhashes are federated when available. - Pinned posts federation From b11dbbf4037143c975c51f4ab6286378b7e74f8e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 17 Aug 2021 18:32:32 -0500 Subject: [PATCH 16/25] UserController: fall back to frontend when user isn't found --- lib/pleroma/web/feed/user_controller.ex | 2 ++ .../pleroma/web/feed/user_controller_test.exs | 19 ++++++++++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex index fa7879caf..739b1f026 100644 --- a/lib/pleroma/web/feed/user_controller.ex +++ b/lib/pleroma/web/feed/user_controller.ex @@ -18,6 +18,8 @@ defmodule Pleroma.Web.Feed.UserController do def feed_redirect(%{assigns: %{format: "html"}} = conn, %{"nickname" => nickname}) do with {_, %User{} = user} <- {:fetch_user, User.get_cached_by_nickname_or_id(nickname)} do Pleroma.Web.Fallback.RedirectController.redirector_with_meta(conn, %{user: user}) + else + _ -> Pleroma.Web.Fallback.RedirectController.redirector(conn, nil) end end diff --git a/test/pleroma/web/feed/user_controller_test.exs b/test/pleroma/web/feed/user_controller_test.exs index 6f6ff433f..6e3f790b2 100644 --- a/test/pleroma/web/feed/user_controller_test.exs +++ b/test/pleroma/web/feed/user_controller_test.exs @@ -196,13 +196,26 @@ test "with html format, it redirects to user feed", %{conn: conn} do ).resp_body end - test "with html format, it returns error when user is not found", %{conn: conn} do + test "with html format, it falls back to frontend when user is remote", %{conn: conn} do + user = insert(:user, local: false) + + {:ok, _} = CommonAPI.post(user, %{status: "test"}) + + response = + conn + |> get("/users/#{user.nickname}") + |> response(200) + + assert response =~ "" + end + + test "with html format, it falls back to frontend when user is not found", %{conn: conn} do response = conn |> get("/users/jimm") - |> json_response(404) + |> response(200) - assert response == %{"error" => "Not found"} + assert response =~ "" end test "with non-html / non-json format, it redirects to user feed in atom format", %{ From e4ad4f0bd6bfceb2791e2c23146552aa981e70bd Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 10 Nov 2021 00:53:32 +0100 Subject: [PATCH 17/25] mix: Update http_signatures to 0.1.1 --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index f91e0b007..1998e89fa 100644 --- a/mix.exs +++ b/mix.exs @@ -158,7 +158,7 @@ defp deps do {:timex, "~> 3.6"}, {:ueberauth, "~> 0.4"}, {:linkify, "~> 0.5.1"}, - {:http_signatures, "~> 0.1.0"}, + {:http_signatures, "~> 0.1.1"}, {:telemetry, "~> 0.3"}, {:poolboy, "~> 1.5"}, {:prometheus, "~> 4.6"}, diff --git a/mix.lock b/mix.lock index 2e39ede30..e45fa28a3 100644 --- a/mix.lock +++ b/mix.lock @@ -58,7 +58,7 @@ "hackney": {:hex, :hackney, "1.17.4", "99da4674592504d3fb0cfef0db84c3ba02b4508bae2dff8c0108baa0d6e0977c", [:rebar3], [{:certifi, "~>2.6.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "de16ff4996556c8548d512f4dbe22dd58a587bf3332e7fd362430a7ef3986b16"}, "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, - "http_signatures": {:hex, :http_signatures, "0.1.0", "4e4b501a936dbf4cb5222597038a89ea10781776770d2e185849fa829686b34c", [:mix], [], "hexpm", "f8a7b3731e3fd17d38fa6e343fcad7b03d6874a3b0a108c8568a71ed9c2cf824"}, + "http_signatures": {:hex, :http_signatures, "0.1.1", "ca7ebc1b61542b163644c8c3b1f0e0f41037d35f2395940d3c6c7deceab41fd8", [:mix], [], "hexpm", "cc3b8a007322cc7b624c0c15eec49ee58ac977254ff529a3c482f681465942a3"}, "httpoison": {:hex, :httpoison, "1.8.0", "6b85dea15820b7804ef607ff78406ab449dd78bed923a49c7160e1886e987a3d", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "28089eaa98cf90c66265b6b5ad87c59a3729bea2e74e9d08f9b51eb9729b3c3a"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "inet_cidr": {:hex, :inet_cidr, "1.0.4", "a05744ab7c221ca8e395c926c3919a821eb512e8f36547c062f62c4ca0cf3d6e", [:mix], [], "hexpm", "64a2d30189704ae41ca7dbdd587f5291db5d1dda1414e0774c29ffc81088c1bc"}, From 23e91ec8ddb143353b5f685feec33c64370d13c6 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 10 Nov 2021 00:37:27 +0100 Subject: [PATCH 18/25] activity_pub_controller: Fix misleading debug warning in post_inbox_fallback --- .../activity_pub/activity_pub_controller.ex | 37 +++++++++---------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 57ac40b42..4a19938f6 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -283,15 +283,29 @@ def inbox(%{assigns: %{valid_signature: true}} = conn, params) do json(conn, "ok") end + def inbox(%{assigns: %{valid_signature: false}} = conn, _params) do + conn + |> put_status(:bad_request) + |> json("Invalid HTTP Signature") + end + # POST /relay/inbox -or- POST /internal/fetch/inbox - def inbox(conn, params) do - if params["type"] == "Create" && FederatingPlug.federating?() do + def inbox(conn, %{"type" => "Create"} = params) do + if FederatingPlug.federating?() do post_inbox_relayed_create(conn, params) else - post_inbox_fallback(conn, params) + conn + |> put_status(:bad_request) + |> json("Not federating") end end + def inbox(conn, _params) do + conn + |> put_status(:bad_request) + |> json("error, missing HTTP Signature") + end + defp post_inbox_relayed_create(conn, params) do Logger.debug( "Signature missing or not from author, relayed Create message, fetching object from source" @@ -302,23 +316,6 @@ defp post_inbox_relayed_create(conn, params) do json(conn, "ok") end - defp post_inbox_fallback(conn, params) do - headers = Enum.into(conn.req_headers, %{}) - - if headers["signature"] && params["actor"] && - String.contains?(headers["signature"], params["actor"]) do - Logger.debug( - "Signature validation error for: #{params["actor"]}, make sure you are forwarding the HTTP Host header!" - ) - - Logger.debug(inspect(conn.req_headers)) - end - - conn - |> put_status(:bad_request) - |> json(dgettext("errors", "error")) - end - defp represent_service_actor(%User{} = user, conn) do with {:ok, user} <- User.ensure_keys_present(user) do conn From 26450a0be3c17e3869d5c990293368091b5ad3a0 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 14 Nov 2021 11:43:39 +0100 Subject: [PATCH 19/25] Mix: Upgrade mogrify library The old library would modify a file on reading it, the new one doesn't have this issue. --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index f91e0b007..91e70f238 100644 --- a/mix.exs +++ b/mix.exs @@ -138,7 +138,7 @@ defp deps do {:cowlib, "~> 2.9", override: true}, {:gun, "~> 2.0.0-rc.1", override: true}, {:jason, "~> 1.2"}, - {:mogrify, "~> 0.7.4"}, + {:mogrify, "~> 0.9.1"}, {:ex_aws, "~> 2.1.6"}, {:ex_aws_s3, "~> 2.0"}, {:sweet_xml, "~> 0.6.6"}, diff --git a/mix.lock b/mix.lock index 2e39ede30..42aeda31f 100644 --- a/mix.lock +++ b/mix.lock @@ -78,7 +78,7 @@ "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, "mochiweb": {:hex, :mochiweb, "2.18.0", "eb55f1db3e6e960fac4e6db4e2db9ec3602cc9f30b86cd1481d56545c3145d2e", [:rebar3], [], "hexpm"}, "mock": {:hex, :mock, "0.3.7", "75b3bbf1466d7e486ea2052a73c6e062c6256fb429d6797999ab02fa32f29e03", [:mix], [{:meck, "~> 0.9.2", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "4da49a4609e41fd99b7836945c26f373623ea968cfb6282742bcb94440cf7e5c"}, - "mogrify": {:hex, :mogrify, "0.7.4", "9b2496dde44b1ce12676f85d7dc531900939e6367bc537c7243a1b089435b32d", [:mix], [], "hexpm", "50d79e337fba6bc95bfbef918058c90f50b17eed9537771e61d4619488f099c3"}, + "mogrify": {:hex, :mogrify, "0.9.1", "a26f107c4987477769f272bd0f7e3ac4b7b75b11ba597fd001b877beffa9c068", [:mix], [], "hexpm", "134edf189337d2125c0948bf0c228fdeef975c594317452d536224069a5b7f05"}, "mox": {:hex, :mox, "1.0.0", "4b3c7005173f47ff30641ba044eb0fe67287743eec9bd9545e37f3002b0a9f8b", [:mix], [], "hexpm", "201b0a20b7abdaaab083e9cf97884950f8a30a1350a1da403b3145e213c6f4df"}, "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "ad0097e2f61d4953bfef20fb6abddf23b87111e6", [ref: "ad0097e2f61d4953bfef20fb6abddf23b87111e6", submodules: true]}, "nimble_parsec": {:hex, :nimble_parsec, "0.5.0", "90e2eca3d0266e5c53f8fbe0079694740b9c91b6747f2b7e3c5d21966bba8300", [:mix], [], "hexpm", "5c040b8469c1ff1b10093d3186e2e10dbe483cd73d79ec017993fb3985b8a9b3"}, From 2dea4a8c041e48bbae2ec6f201b4c0e95b8131ae Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 14 Nov 2021 11:44:24 +0100 Subject: [PATCH 20/25] StealEmojiPolicyTest: Make mocks explicit. --- .../activity_pub/mrf/steal_emoji_policy_test.exs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs b/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs index bae57f29a..1b37e4c26 100644 --- a/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs @@ -9,11 +9,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do alias Pleroma.Emoji alias Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy - setup_all do - Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) - :ok - end - setup do emoji_path = [:instance, :static_dir] |> Config.get() |> Path.join("emoji/stolen") @@ -49,6 +44,10 @@ test "Steals emoji on unknown shortcode from allowed remote host", %{ refute "firedfox" in installed() refute File.exists?(path) + Tesla.Mock.mock(fn %{method: :get, url: "https://example.org/emoji/firedfox.png"} -> + %Tesla.Env{status: 200, body: File.read!("test/fixtures/image.jpg")} + end) + clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 284_468) assert {:ok, _message} = StealEmojiPolicy.filter(message) @@ -78,6 +77,10 @@ test "reject shortcode", %{message: message} do test "reject if size is above the limit", %{message: message} do refute "firedfox" in installed() + Tesla.Mock.mock(fn %{method: :get, url: "https://example.org/emoji/firedfox.png"} -> + %Tesla.Env{status: 200, body: File.read!("test/fixtures/image.jpg")} + end) + clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 50_000) assert {:ok, _message} = StealEmojiPolicy.filter(message) From 871936b3ca800809c616147bc149c4ef4effed69 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 14 Nov 2021 11:45:38 +0100 Subject: [PATCH 21/25] MediaProxyCacheControllerTest: Fix unstable tests. Cachex order isn't stable, so we can not depend on the order in tests. This adds some sorting to make the tests stable. --- .../media_proxy_cache_controller_test.exs | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/test/pleroma/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/pleroma/web/admin_api/controllers/media_proxy_cache_controller_test.exs index 5d872901e..1818c8a8e 100644 --- a/test/pleroma/web/admin_api/controllers/media_proxy_cache_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/media_proxy_cache_controller_test.exs @@ -47,30 +47,34 @@ test "shows banned MediaProxy URLs", %{conn: conn} do assert response["page_size"] == 2 assert response["count"] == 5 - assert response["urls"] == [ - "http://localhost:4001/media/fb1f4d.jpg", - "http://localhost:4001/media/a688346.jpg" - ] + results = response["urls"] response = conn |> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=2") |> json_response_and_validate_schema(200) - assert response["urls"] == [ - "http://localhost:4001/media/gb1f44.jpg", - "http://localhost:4001/media/tb13f47.jpg" - ] - assert response["page_size"] == 2 assert response["count"] == 5 + results = results ++ response["urls"] + response = conn |> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=3") |> json_response_and_validate_schema(200) - assert response["urls"] == ["http://localhost:4001/media/wb1f46.jpg"] + results = results ++ response["urls"] + + assert results |> Enum.sort() == + [ + "http://localhost:4001/media/wb1f46.jpg", + "http://localhost:4001/media/gb1f44.jpg", + "http://localhost:4001/media/tb13f47.jpg", + "http://localhost:4001/media/fb1f4d.jpg", + "http://localhost:4001/media/a688346.jpg" + ] + |> Enum.sort() end test "search banned MediaProxy URLs", %{conn: conn} do @@ -88,9 +92,9 @@ test "search banned MediaProxy URLs", %{conn: conn} do |> get("/api/pleroma/admin/media_proxy_caches?page_size=2&query=F44") |> json_response_and_validate_schema(200) - assert response["urls"] == [ - "http://localhost:4001/media/gb1f44.jpg", - "http://localhost:4001/media/ff44b1f4d.jpg" + assert response["urls"] |> Enum.sort() == [ + "http://localhost:4001/media/ff44b1f4d.jpg", + "http://localhost:4001/media/gb1f44.jpg" ] assert response["page_size"] == 2 From 7dcc73f2944601df6021b5ea41672f0eda999524 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 14 Nov 2021 16:04:08 +0100 Subject: [PATCH 22/25] Mix: Update crypt to fix musl builds. --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index f91e0b007..3339efe4e 100644 --- a/mix.exs +++ b/mix.exs @@ -146,7 +146,7 @@ defp deps do {:bbcode_pleroma, "~> 0.2.0"}, {:crypt, git: "https://github.com/msantos/crypt.git", - ref: "031d812ea45593de598bdbc3d28bf45a6e976de3"}, + ref: "f75cd55325e33cbea198fb41fe41871392f8fb76"}, {:cors_plug, "~> 2.0"}, {:web_push_encryption, git: "https://github.com/lanodan/elixir-web-push-encryption.git", branch: "bugfix/otp-24"}, diff --git a/mix.lock b/mix.lock index 2e39ede30..5d2255564 100644 --- a/mix.lock +++ b/mix.lock @@ -22,7 +22,7 @@ "cowlib": {:hex, :cowlib, "2.11.0", "0b9ff9c346629256c42ebe1eeb769a83c6cb771a6ee5960bd110ab0b9b872063", [:make, :rebar3], [], "hexpm", "2b3e9da0b21c4565751a6d4901c20d1b4cc25cbb7fd50d91d2ab6dd287bc86a9"}, "credo": {:hex, :credo, "1.5.5", "e8f422026f553bc3bebb81c8e8bf1932f498ca03339856c7fec63d3faac8424b", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "dd8623ab7091956a855dc9f3062486add9c52d310dfd62748779c4315d8247de"}, "crontab": {:hex, :crontab, "1.1.8", "2ce0e74777dfcadb28a1debbea707e58b879e6aa0ffbf9c9bb540887bce43617", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, - "crypt": {:git, "https://github.com/msantos/crypt.git", "031d812ea45593de598bdbc3d28bf45a6e976de3", [ref: "031d812ea45593de598bdbc3d28bf45a6e976de3"]}, + "crypt": {:git, "https://github.com/msantos/crypt.git", "f75cd55325e33cbea198fb41fe41871392f8fb76", [ref: "f75cd55325e33cbea198fb41fe41871392f8fb76"]}, "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"}, "db_connection": {:hex, :db_connection, "2.4.0", "d04b1b73795dae60cead94189f1b8a51cc9e1f911c234cc23074017c43c031e5", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ad416c21ad9f61b3103d254a71b63696ecadb6a917b36f563921e0de00d7d7c8"}, "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, From 5c7aa4a1e999bdbbda72529c1ed5b92152aa3e1a Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 14 Nov 2021 20:37:32 +0100 Subject: [PATCH 23/25] CI: Conservatively update release images so they keep building. --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9041443c5..c3192fbf2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -298,7 +298,7 @@ arm: only: *release-only tags: - arm32-specified - image: arm32v7/elixir:1.10.3 + image: arm32v7/elixir:1.10.4 cache: *release-cache variables: *release-variables before_script: *before-release @@ -322,7 +322,7 @@ arm64: only: *release-only tags: - arm - image: arm64v8/elixir:1.10.3 + image: arm64v8/elixir:1.10.4 cache: *release-cache variables: *release-variables before_script: *before-release From f78cb6ab9af55d4c9c39f67e9ab0f7c70453d0eb Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 15 Nov 2021 21:04:43 +0100 Subject: [PATCH 24/25] CI: Upload the image for all platforms For some reason the other platforms started failing, too. This fixes it. Why? Don't know. --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c3192fbf2..844f5888e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -244,7 +244,7 @@ stop_review_app: amd64: stage: release - image: elixir:1.10.3 + image: elixir:1.10.4 only: &release-only - stable@pleroma/pleroma - develop@pleroma/pleroma @@ -282,7 +282,7 @@ amd64-musl: stage: release artifacts: *release-artifacts only: *release-only - image: elixir:1.10.3-alpine + image: elixir:1.10.4-alpine cache: *release-cache variables: *release-variables before_script: &before-release-musl @@ -310,7 +310,7 @@ arm-musl: only: *release-only tags: - arm32-specified - image: arm32v7/elixir:1.10.3-alpine + image: arm32v7/elixir:1.10.4-alpine cache: *release-cache variables: *release-variables before_script: *before-release-musl @@ -334,7 +334,7 @@ arm64-musl: only: *release-only tags: - arm - image: arm64v8/elixir:1.10.3-alpine + image: arm64v8/elixir:1.10.4-alpine cache: *release-cache variables: *release-variables before_script: *before-release-musl From 5e15ceb49d9accadce01732fbec7dd4f22ebdfa3 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 16 Nov 2021 14:32:49 +0100 Subject: [PATCH 25/25] mix: Update earmark This should fix the presence of extraneous newlines in markdown output. Related: https://github.com/pragdave/earmark/issues/406 --- mix.exs | 2 +- mix.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mix.exs b/mix.exs index f91e0b007..a9350789d 100644 --- a/mix.exs +++ b/mix.exs @@ -142,7 +142,7 @@ defp deps do {:ex_aws, "~> 2.1.6"}, {:ex_aws_s3, "~> 2.0"}, {:sweet_xml, "~> 0.6.6"}, - {:earmark, "1.4.15"}, + {:earmark, "~> 1.4.15"}, {:bbcode_pleroma, "~> 0.2.0"}, {:crypt, git: "https://github.com/msantos/crypt.git", diff --git a/mix.lock b/mix.lock index 2e39ede30..54beb88b7 100644 --- a/mix.lock +++ b/mix.lock @@ -27,8 +27,8 @@ "db_connection": {:hex, :db_connection, "2.4.0", "d04b1b73795dae60cead94189f1b8a51cc9e1f911c234cc23074017c43c031e5", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ad416c21ad9f61b3103d254a71b63696ecadb6a917b36f563921e0de00d7d7c8"}, "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, - "earmark": {:hex, :earmark, "1.4.15", "2c7f924bf495ec1f65bd144b355d0949a05a254d0ec561740308a54946a67888", [:mix], [{:earmark_parser, ">= 1.4.13", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "3b1209b85bc9f3586f370f7c363f6533788fb4e51db23aa79565875e7f9999ee"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.13", "0c98163e7d04a15feb62000e1a891489feb29f3d10cb57d4f845c405852bbef8", [:mix], [], "hexpm", "d602c26af3a0af43d2f2645613f65841657ad6efc9f0e361c3b6c06b578214ba"}, + "earmark": {:hex, :earmark, "1.4.18", "618c4ff1563450d1832b7fb41dc6755e470f91a6fd4c70f350a58b14f64a7db8", [:mix], [{:earmark_parser, ">= 1.4.17", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "57ac3b6da3958ed09c669a9b159e86377fcccda56bacde8a209fa4dcdef52560"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.17", "6f3c7e94170377ba45241d394389e800fb15adc5de51d0a3cd52ae766aafd63f", [:mix], [], "hexpm", "f93ac89c9feca61c165b264b5837bf82344d13bebc634cd575cb711e2e342023"}, "eblurhash": {:hex, :eblurhash, "1.1.0", "e10ccae762598507ebfacf0b645ed49520f2afa3e7e9943e73a91117dffce415", [:rebar3], [], "hexpm", "2e6b889d09fddd374e3c5ac57c486138768763264e99ac1074ae5fa7fc9ab51d"}, "ecto": {:hex, :ecto, "3.6.2", "efdf52acfc4ce29249bab5417415bd50abd62db7b0603b8bab0d7b996548c2bc", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "efad6dfb04e6f986b8a3047822b0f826d9affe8e4ebdd2aeedbfcb14fd48884e"}, "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"},