From 0d5088c2b83fafd9d8da1f1b04936f831ac5ee87 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 1 Sep 2020 09:37:08 +0300 Subject: [PATCH 01/75] remove `unread_conversation_count` from User --- lib/pleroma/conversation.ex | 6 +-- lib/pleroma/conversation/participation.ex | 27 +++++------- lib/pleroma/user.ex | 42 ------------------- .../web/mastodon_api/views/account_view.ex | 2 +- ...ve_unread_conversation_count_from_user.exs | 38 +++++++++++++++++ ...ad_index_to_conversation_participation.exs | 12 ++++++ test/conversation/participation_test.exs | 32 +++++++------- .../conversation_controller_test.exs | 23 +++++----- .../conversation_controller_test.exs | 4 +- 9 files changed, 92 insertions(+), 94 deletions(-) create mode 100644 priv/repo/migrations/20200831114918_remove_unread_conversation_count_from_user.exs create mode 100644 priv/repo/migrations/20200831115854_add_unread_index_to_conversation_participation.exs diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index e76eb0087..77933f0be 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -43,7 +43,7 @@ def get_for_ap_id(ap_id) do def maybe_create_recipientships(participation, activity) do participation = Repo.preload(participation, :recipients) - if participation.recipients |> Enum.empty?() do + if Enum.empty?(participation.recipients) do recipients = User.get_all_by_ap_id(activity.recipients) RecipientShip.create(recipients, participation) end @@ -69,10 +69,6 @@ def create_or_bump_for(activity, opts \\ []) do Enum.map(users, fn user -> invisible_conversation = Enum.any?(users, &User.blocks?(user, &1)) - unless invisible_conversation do - User.increment_unread_conversation_count(conversation, user) - end - opts = Keyword.put(opts, :invisible_conversation, invisible_conversation) {:ok, participation} = diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index 8bc3e85d6..4c32b273a 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -63,21 +63,10 @@ def mark_as_read(%User{} = user, %Conversation{} = conversation) do end end - def mark_as_read(participation) do - __MODULE__ - |> where(id: ^participation.id) - |> update(set: [read: true]) - |> select([p], p) - |> Repo.update_all([]) - |> case do - {1, [participation]} -> - participation = Repo.preload(participation, :user) - User.set_unread_conversation_count(participation.user) - {:ok, participation} - - error -> - error - end + def mark_as_read(%__MODULE__{} = participation) do + participation + |> change(read: true) + |> Repo.update() end def mark_all_as_read(%User{local: true} = user, %User{} = target_user) do @@ -93,7 +82,6 @@ def mark_all_as_read(%User{local: true} = user, %User{} = target_user) do |> update([p], set: [read: true]) |> Repo.update_all([]) - {:ok, user} = User.set_unread_conversation_count(user) {:ok, user, []} end @@ -108,7 +96,6 @@ def mark_all_as_read(%User{} = user) do |> select([p], p) |> Repo.update_all([]) - {:ok, user} = User.set_unread_conversation_count(user) {:ok, user, participations} end @@ -220,6 +207,12 @@ def set_recipients(participation, user_ids) do {:ok, Repo.preload(participation, :recipients, force: true)} end + @spec unread_count(User.t()) :: integer() + def unread_count(%User{id: user_id}) do + from(q in __MODULE__, where: q.user_id == ^user_id and q.read == false) + |> Repo.aggregate(:count, :id) + end + def unread_conversation_count_for_user(user) do from(p in __MODULE__, where: p.user_id == ^user.id, diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index d2ad9516f..7fc7a533e 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -129,7 +129,6 @@ defmodule Pleroma.User do field(:hide_followers, :boolean, default: false) field(:hide_follows, :boolean, default: false) field(:hide_favorites, :boolean, default: true) - field(:unread_conversation_count, :integer, default: 0) field(:pinned_activities, {:array, :string}, default: []) field(:email_notifications, :map, default: %{"digest" => false}) field(:mascot, :map, default: nil) @@ -1295,47 +1294,6 @@ def update_following_count(%User{local: true} = user) do |> update_and_set_cache() end - def set_unread_conversation_count(%User{local: true} = user) do - unread_query = Participation.unread_conversation_count_for_user(user) - - User - |> join(:inner, [u], p in subquery(unread_query)) - |> update([u, p], - set: [unread_conversation_count: p.count] - ) - |> where([u], u.id == ^user.id) - |> select([u], u) - |> Repo.update_all([]) - |> case do - {1, [user]} -> set_cache(user) - _ -> {:error, user} - end - end - - def set_unread_conversation_count(user), do: {:ok, user} - - def increment_unread_conversation_count(conversation, %User{local: true} = user) do - unread_query = - Participation.unread_conversation_count_for_user(user) - |> where([p], p.conversation_id == ^conversation.id) - - User - |> join(:inner, [u], p in subquery(unread_query)) - |> update([u, p], - inc: [unread_conversation_count: 1] - ) - |> where([u], u.id == ^user.id) - |> where([u, p], p.count == 0) - |> select([u], u) - |> Repo.update_all([]) - |> case do - {1, [user]} -> set_cache(user) - _ -> {:error, user} - end - end - - def increment_unread_conversation_count(_, user), do: {:ok, user} - @spec get_users_from_set([String.t()], keyword()) :: [User.t()] def get_users_from_set(ap_ids, opts \\ []) do local_only = Keyword.get(opts, :local_only, true) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 864c0417f..1bf53600c 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -386,7 +386,7 @@ defp maybe_put_unread_conversation_count(data, %User{id: user_id} = user, %User{ data |> Kernel.put_in( [:pleroma, :unread_conversation_count], - user.unread_conversation_count + Pleroma.Conversation.Participation.unread_count(user) ) end diff --git a/priv/repo/migrations/20200831114918_remove_unread_conversation_count_from_user.exs b/priv/repo/migrations/20200831114918_remove_unread_conversation_count_from_user.exs new file mode 100644 index 000000000..b7bdb9166 --- /dev/null +++ b/priv/repo/migrations/20200831114918_remove_unread_conversation_count_from_user.exs @@ -0,0 +1,38 @@ +defmodule Pleroma.Repo.Migrations.RemoveUnreadConversationCountFromUser do + use Ecto.Migration + import Ecto.Query + alias Pleroma.Repo + + def up do + alter table(:users) do + remove_if_exists(:unread_conversation_count, :integer) + end + end + + def down do + alter table(:users) do + add_if_not_exists(:unread_conversation_count, :integer, default: 0) + end + + flush() + recalc_unread_conversation_count() + end + + defp recalc_unread_conversation_count do + participations_subquery = + from( + p in "conversation_participations", + where: p.read == false, + group_by: p.user_id, + select: %{user_id: p.user_id, unread_conversation_count: count(p.id)} + ) + + from( + u in "users", + join: p in subquery(participations_subquery), + on: p.user_id == u.id, + update: [set: [unread_conversation_count: p.unread_conversation_count]] + ) + |> Repo.update_all([]) + end +end diff --git a/priv/repo/migrations/20200831115854_add_unread_index_to_conversation_participation.exs b/priv/repo/migrations/20200831115854_add_unread_index_to_conversation_participation.exs new file mode 100644 index 000000000..68771c655 --- /dev/null +++ b/priv/repo/migrations/20200831115854_add_unread_index_to_conversation_participation.exs @@ -0,0 +1,12 @@ +defmodule Pleroma.Repo.Migrations.AddUnreadIndexToConversationParticipation do + use Ecto.Migration + + def change do + create( + index(:conversation_participations, [:user_id], + where: "read = false", + name: "unread_conversation_participation_count_index" + ) + ) + end +end diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index 59a1b6492..5a603dcc1 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -37,9 +37,8 @@ test "for a new conversation or a reply, it doesn't mark the author's participat [%{read: true}] = Participation.for_user(user) [%{read: false} = participation] = Participation.for_user(other_user) - - assert User.get_cached_by_id(user.id).unread_conversation_count == 0 - assert User.get_cached_by_id(other_user.id).unread_conversation_count == 1 + assert Participation.unread_count(user) == 0 + assert Participation.unread_count(other_user) == 1 {:ok, _} = CommonAPI.post(other_user, %{ @@ -54,8 +53,8 @@ test "for a new conversation or a reply, it doesn't mark the author's participat [%{read: false}] = Participation.for_user(user) [%{read: true}] = Participation.for_user(other_user) - assert User.get_cached_by_id(user.id).unread_conversation_count == 1 - assert User.get_cached_by_id(other_user.id).unread_conversation_count == 0 + assert Participation.unread_count(user) == 1 + assert Participation.unread_count(other_user) == 0 end test "for a new conversation, it sets the recipents of the participation" do @@ -264,7 +263,7 @@ test "when the user blocks a recipient, the existing conversations with them are assert [%{read: false}, %{read: false}, %{read: false}, %{read: false}] = Participation.for_user(blocker) - assert User.get_cached_by_id(blocker.id).unread_conversation_count == 4 + assert Participation.unread_count(blocker) == 4 {:ok, _user_relationship} = User.block(blocker, blocked) @@ -272,15 +271,15 @@ test "when the user blocks a recipient, the existing conversations with them are assert [%{read: true}, %{read: true}, %{read: true}, %{read: false}] = Participation.for_user(blocker) - assert User.get_cached_by_id(blocker.id).unread_conversation_count == 1 + assert Participation.unread_count(blocker) == 1 # The conversation is not marked as read for the blocked user assert [_, _, %{read: false}] = Participation.for_user(blocked) - assert User.get_cached_by_id(blocked.id).unread_conversation_count == 1 + assert Participation.unread_count(blocker) == 1 # The conversation is not marked as read for the third user assert [%{read: false}, _, _] = Participation.for_user(third_user) - assert User.get_cached_by_id(third_user.id).unread_conversation_count == 1 + assert Participation.unread_count(third_user) == 1 end test "the new conversation with the blocked user is not marked as unread " do @@ -298,7 +297,7 @@ test "the new conversation with the blocked user is not marked as unread " do }) assert [%{read: true}] = Participation.for_user(blocker) - assert User.get_cached_by_id(blocker.id).unread_conversation_count == 0 + assert Participation.unread_count(blocker) == 0 # When the blocked user is a recipient {:ok, _direct2} = @@ -308,10 +307,10 @@ test "the new conversation with the blocked user is not marked as unread " do }) assert [%{read: true}, %{read: true}] = Participation.for_user(blocker) - assert User.get_cached_by_id(blocker.id).unread_conversation_count == 0 + assert Participation.unread_count(blocker) == 0 assert [%{read: false}, _] = Participation.for_user(blocked) - assert User.get_cached_by_id(blocked.id).unread_conversation_count == 1 + assert Participation.unread_count(blocked) == 1 end test "the conversation with the blocked user is not marked as unread on a reply" do @@ -327,8 +326,8 @@ test "the conversation with the blocked user is not marked as unread on a reply" {:ok, _user_relationship} = User.block(blocker, blocked) assert [%{read: true}] = Participation.for_user(blocker) - assert User.get_cached_by_id(blocker.id).unread_conversation_count == 0 + assert Participation.unread_count(blocker) == 0 assert [blocked_participation] = Participation.for_user(blocked) # When it's a reply from the blocked user @@ -340,8 +339,8 @@ test "the conversation with the blocked user is not marked as unread on a reply" }) assert [%{read: true}] = Participation.for_user(blocker) - assert User.get_cached_by_id(blocker.id).unread_conversation_count == 0 + assert Participation.unread_count(blocker) == 0 assert [third_user_participation] = Participation.for_user(third_user) # When it's a reply from the third user @@ -353,11 +352,12 @@ test "the conversation with the blocked user is not marked as unread on a reply" }) assert [%{read: true}] = Participation.for_user(blocker) - assert User.get_cached_by_id(blocker.id).unread_conversation_count == 0 + assert Participation.unread_count(blocker) == 0 # Marked as unread for the blocked user assert [%{read: false}] = Participation.for_user(blocked) - assert User.get_cached_by_id(blocked.id).unread_conversation_count == 1 + + assert Participation.unread_count(blocked) == 1 end end end diff --git a/test/web/mastodon_api/controllers/conversation_controller_test.exs b/test/web/mastodon_api/controllers/conversation_controller_test.exs index 3e21e6bf1..b23b22752 100644 --- a/test/web/mastodon_api/controllers/conversation_controller_test.exs +++ b/test/web/mastodon_api/controllers/conversation_controller_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.Web.MastodonAPI.ConversationControllerTest do use Pleroma.Web.ConnCase + alias Pleroma.Conversation.Participation alias Pleroma.User alias Pleroma.Web.CommonAPI @@ -28,10 +29,10 @@ test "returns correct conversations", %{ user_three: user_three, conn: conn } do - assert User.get_cached_by_id(user_two.id).unread_conversation_count == 0 + assert Participation.unread_count(user_two) == 0 {:ok, direct} = create_direct_message(user_one, [user_two, user_three]) - assert User.get_cached_by_id(user_two.id).unread_conversation_count == 1 + assert Participation.unread_count(user_two) == 1 {:ok, _follower_only} = CommonAPI.post(user_one, %{ @@ -59,7 +60,7 @@ test "returns correct conversations", %{ assert is_binary(res_id) assert unread == false assert res_last_status["id"] == direct.id - assert User.get_cached_by_id(user_one.id).unread_conversation_count == 0 + assert Participation.unread_count(user_one) == 0 end test "observes limit params", %{ @@ -134,8 +135,8 @@ test "the user marks a conversation as read", %{user: user_one, conn: conn} do user_two = insert(:user) {:ok, direct} = create_direct_message(user_one, [user_two]) - assert User.get_cached_by_id(user_one.id).unread_conversation_count == 0 - assert User.get_cached_by_id(user_two.id).unread_conversation_count == 1 + assert Participation.unread_count(user_one) == 0 + assert Participation.unread_count(user_two) == 1 user_two_conn = build_conn() @@ -155,8 +156,8 @@ test "the user marks a conversation as read", %{user: user_one, conn: conn} do |> post("/api/v1/conversations/#{direct_conversation_id}/read") |> json_response_and_validate_schema(200) - assert User.get_cached_by_id(user_one.id).unread_conversation_count == 0 - assert User.get_cached_by_id(user_two.id).unread_conversation_count == 0 + assert Participation.unread_count(user_one) == 0 + assert Participation.unread_count(user_two) == 0 # The conversation is marked as unread on reply {:ok, _} = @@ -171,8 +172,8 @@ test "the user marks a conversation as read", %{user: user_one, conn: conn} do |> get("/api/v1/conversations") |> json_response_and_validate_schema(200) - assert User.get_cached_by_id(user_one.id).unread_conversation_count == 1 - assert User.get_cached_by_id(user_two.id).unread_conversation_count == 0 + assert Participation.unread_count(user_one) == 1 + assert Participation.unread_count(user_two) == 0 # A reply doesn't increment the user's unread_conversation_count if the conversation is unread {:ok, _} = @@ -182,8 +183,8 @@ test "the user marks a conversation as read", %{user: user_one, conn: conn} do in_reply_to_status_id: direct.id }) - assert User.get_cached_by_id(user_one.id).unread_conversation_count == 1 - assert User.get_cached_by_id(user_two.id).unread_conversation_count == 0 + assert Participation.unread_count(user_one) == 1 + assert Participation.unread_count(user_two) == 0 end test "(vanilla) Mastodon frontend behaviour", %{user: user_one, conn: conn} do diff --git a/test/web/pleroma_api/controllers/conversation_controller_test.exs b/test/web/pleroma_api/controllers/conversation_controller_test.exs index e6d0b3e37..f2feeaaef 100644 --- a/test/web/pleroma_api/controllers/conversation_controller_test.exs +++ b/test/web/pleroma_api/controllers/conversation_controller_test.exs @@ -121,7 +121,7 @@ test "POST /api/v1/pleroma/conversations/read" do [participation2, participation1] = Participation.for_user(other_user) assert Participation.get(participation2.id).read == false assert Participation.get(participation1.id).read == false - assert User.get_cached_by_id(other_user.id).unread_conversation_count == 2 + assert Participation.unread_count(other_user) == 2 [%{"unread" => false}, %{"unread" => false}] = conn @@ -131,6 +131,6 @@ test "POST /api/v1/pleroma/conversations/read" do [participation2, participation1] = Participation.for_user(other_user) assert Participation.get(participation2.id).read == true assert Participation.get(participation1.id).read == true - assert User.get_cached_by_id(other_user.id).unread_conversation_count == 0 + assert Participation.unread_count(other_user) == 0 end end From 7efadc3cbd46369e960f31c33a2c555f718ca8c5 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 1 Oct 2020 21:34:45 +0300 Subject: [PATCH 02/75] No auth check in OStatusController, even on non-federating instances. --- lib/pleroma/web/ostatus/ostatus_controller.ex | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index de1b0b3f0..8646d2c1c 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -16,10 +16,6 @@ defmodule Pleroma.Web.OStatus.OStatusController do alias Pleroma.Web.Metadata.PlayerView alias Pleroma.Web.Router - plug(Pleroma.Plugs.EnsureAuthenticatedPlug, - unless_func: &Pleroma.Web.FederatingPlug.federating?/1 - ) - plug( RateLimiter, [name: :ap_routes, params: ["uuid"]] when action in [:object, :activity] From 0d575735bfd280b878bdecc6d018d8cca23ad09f Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 1 Oct 2020 21:41:22 +0300 Subject: [PATCH 03/75] No auth check in UserController.feed_redirect/2, even on non-federating instances. --- lib/pleroma/web/feed/user_controller.ex | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex index 71eb1ea7e..09ecdedb4 100644 --- a/lib/pleroma/web/feed/user_controller.ex +++ b/lib/pleroma/web/feed/user_controller.ex @@ -23,12 +23,7 @@ def feed_redirect(%{assigns: %{format: "html"}} = conn, %{"nickname" => nickname def feed_redirect(%{assigns: %{format: format}} = conn, _params) when format in ["json", "activity+json"] do - with %{halted: false} = conn <- - Pleroma.Plugs.EnsureAuthenticatedPlug.call(conn, - unless_func: &Pleroma.Web.FederatingPlug.federating?/1 - ) do - ActivityPubController.call(conn, :user) - end + ActivityPubController.call(conn, :user) end def feed_redirect(conn, %{"nickname" => nickname}) do From f6024252ae8601d41bea943bb3cae5c656416eb9 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 2 Oct 2020 22:18:02 +0300 Subject: [PATCH 04/75] [#3053] No auth check in StaticFEController, even on non-federating instances. Adjusted tests. --- lib/pleroma/web/feed/tag_controller.ex | 4 +- lib/pleroma/web/feed/user_controller.ex | 4 +- lib/pleroma/web/router.ex | 11 +- .../web/static_fe/static_fe_controller.ex | 176 +++++++++--------- test/support/conn_case.ex | 22 --- .../activity_pub_controller_test.exs | 19 ++ test/web/feed/user_controller_test.exs | 12 +- test/web/ostatus/ostatus_controller_test.exs | 24 +-- .../static_fe/static_fe_controller_test.exs | 24 ++- 9 files changed, 162 insertions(+), 134 deletions(-) diff --git a/lib/pleroma/web/feed/tag_controller.ex b/lib/pleroma/web/feed/tag_controller.ex index 93a8294b7..c348b32c2 100644 --- a/lib/pleroma/web/feed/tag_controller.ex +++ b/lib/pleroma/web/feed/tag_controller.ex @@ -10,14 +10,14 @@ defmodule Pleroma.Web.Feed.TagController do alias Pleroma.Web.Feed.FeedView def feed(conn, params) do - unless Pleroma.Config.restrict_unauthenticated_access?(:activities, :local) do + unless Config.restrict_unauthenticated_access?(:activities, :local) do render_feed(conn, params) else render_error(conn, :not_found, "Not found") end end - def render_feed(conn, %{"tag" => raw_tag} = params) do + defp render_feed(conn, %{"tag" => raw_tag} = params) do {format, tag} = parse_tag(raw_tag) activities = diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex index 09ecdedb4..5fbcd82d7 100644 --- a/lib/pleroma/web/feed/user_controller.ex +++ b/lib/pleroma/web/feed/user_controller.ex @@ -40,11 +40,11 @@ def feed(conn, params) do end end - def render_feed(conn, %{"nickname" => nickname} = params) do + defp render_feed(conn, %{"nickname" => nickname} = params) do format = get_format(conn) format = - if format in ["rss", "atom"] do + if format in ["atom", "rss"] do format else "atom" diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 42a9db21d..e0e92549f 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -561,12 +561,17 @@ defmodule Pleroma.Web.Router do plug(Pleroma.Plugs.StaticFEPlug) end + pipeline :ostatus_no_html do + plug(:accepts, ["xml", "rss", "atom", "activity+json", "json"]) + end + pipeline :oembed do plug(:accepts, ["json", "xml"]) end scope "/", Pleroma.Web do - pipe_through([:ostatus, :http_signature]) + # Note: no authentication plugs, all endpoints below should only yield public objects + pipe_through(:ostatus) get("/objects/:uuid", OStatus.OStatusController, :object) get("/activities/:uuid", OStatus.OStatusController, :activity) @@ -579,6 +584,10 @@ defmodule Pleroma.Web.Router do get("/users/:nickname/feed", Feed.UserController, :feed, as: :user_feed) get("/users/:nickname", Feed.UserController, :feed_redirect, as: :user_feed) + end + + scope "/", Pleroma.Web do + pipe_through(:ostatus_no_html) get("/tags/:tag", Feed.TagController, :feed, as: :tag_feed) end diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index a7a891b13..b1c62f5b0 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -17,12 +17,95 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do plug(:put_view, Pleroma.Web.StaticFE.StaticFEView) plug(:assign_id) - plug(Pleroma.Plugs.EnsureAuthenticatedPlug, - unless_func: &Pleroma.Web.FederatingPlug.federating?/1 - ) - @page_keys ["max_id", "min_id", "limit", "since_id", "order"] + @doc "Renders requested local public activity" + def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do + with %Activity{local: true} = activity <- + Activity.get_by_id_with_object(notice_id), + true <- Visibility.is_public?(activity.object), + %User{} = user <- User.get_by_ap_id(activity.object.data["actor"]) do + meta = Metadata.build_tags(%{activity_id: notice_id, object: activity.object, user: user}) + + timeline = + activity.object.data["context"] + |> ActivityPub.fetch_activities_for_context(%{}) + |> Enum.reverse() + |> Enum.map(&represent(&1, &1.object.id == activity.object.id)) + + render(conn, "conversation.html", %{activities: timeline, meta: meta}) + else + %Activity{object: %Object{data: data}} -> + conn + |> put_status(:found) + |> redirect(external: data["url"] || data["external_url"] || data["id"]) + + _ -> + not_found(conn, "Post not found.") + end + end + + @doc "Renders public activities of requested user" + def show(%{assigns: %{username_or_id: username_or_id}} = conn, params) do + case User.get_cached_by_nickname_or_id(username_or_id) do + %User{} = user -> + meta = Metadata.build_tags(%{user: user}) + + params = + params + |> Map.take(@page_keys) + |> Map.new(fn {k, v} -> {String.to_existing_atom(k), v} end) + + timeline = + user + |> ActivityPub.fetch_user_activities(_reading_user = nil, params) + |> Enum.map(&represent/1) + + prev_page_id = + (params["min_id"] || params["max_id"]) && + List.first(timeline) && List.first(timeline).id + + next_page_id = List.last(timeline) && List.last(timeline).id + + render(conn, "profile.html", %{ + user: User.sanitize_html(user), + timeline: timeline, + prev_page_id: prev_page_id, + next_page_id: next_page_id, + meta: meta + }) + + _ -> + not_found(conn, "User not found.") + end + end + + def show(%{assigns: %{object_id: _}} = conn, _params) do + url = Helpers.url(conn) <> conn.request_path + + case Activity.get_create_by_object_ap_id_with_object(url) do + %Activity{} = activity -> + to = Helpers.o_status_path(Pleroma.Web.Endpoint, :notice, activity) + redirect(conn, to: to) + + _ -> + not_found(conn, "Post not found.") + end + end + + def show(%{assigns: %{activity_id: _}} = conn, _params) do + url = Helpers.url(conn) <> conn.request_path + + case Activity.get_by_ap_id(url) do + %Activity{} = activity -> + to = Helpers.o_status_path(Pleroma.Web.Endpoint, :notice, activity) + redirect(conn, to: to) + + _ -> + not_found(conn, "Post not found.") + end + end + defp get_title(%Object{data: %{"name" => name}}) when is_binary(name), do: name @@ -81,91 +164,6 @@ defp represent(%Activity{object: %Object{data: data}} = activity, selected) do } end - def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do - with %Activity{local: true} = activity <- - Activity.get_by_id_with_object(notice_id), - true <- Visibility.is_public?(activity.object), - %User{} = user <- User.get_by_ap_id(activity.object.data["actor"]) do - meta = Metadata.build_tags(%{activity_id: notice_id, object: activity.object, user: user}) - - timeline = - activity.object.data["context"] - |> ActivityPub.fetch_activities_for_context(%{}) - |> Enum.reverse() - |> Enum.map(&represent(&1, &1.object.id == activity.object.id)) - - render(conn, "conversation.html", %{activities: timeline, meta: meta}) - else - %Activity{object: %Object{data: data}} -> - conn - |> put_status(:found) - |> redirect(external: data["url"] || data["external_url"] || data["id"]) - - _ -> - not_found(conn, "Post not found.") - end - end - - def show(%{assigns: %{username_or_id: username_or_id}} = conn, params) do - case User.get_cached_by_nickname_or_id(username_or_id) do - %User{} = user -> - meta = Metadata.build_tags(%{user: user}) - - params = - params - |> Map.take(@page_keys) - |> Map.new(fn {k, v} -> {String.to_existing_atom(k), v} end) - - timeline = - user - |> ActivityPub.fetch_user_activities(nil, params) - |> Enum.map(&represent/1) - - prev_page_id = - (params["min_id"] || params["max_id"]) && - List.first(timeline) && List.first(timeline).id - - next_page_id = List.last(timeline) && List.last(timeline).id - - render(conn, "profile.html", %{ - user: User.sanitize_html(user), - timeline: timeline, - prev_page_id: prev_page_id, - next_page_id: next_page_id, - meta: meta - }) - - _ -> - not_found(conn, "User not found.") - end - end - - def show(%{assigns: %{object_id: _}} = conn, _params) do - url = Helpers.url(conn) <> conn.request_path - - case Activity.get_create_by_object_ap_id_with_object(url) do - %Activity{} = activity -> - to = Helpers.o_status_path(Pleroma.Web.Endpoint, :notice, activity) - redirect(conn, to: to) - - _ -> - not_found(conn, "Post not found.") - end - end - - def show(%{assigns: %{activity_id: _}} = conn, _params) do - url = Helpers.url(conn) <> conn.request_path - - case Activity.get_by_ap_id(url) do - %Activity{} = activity -> - to = Helpers.o_status_path(Pleroma.Web.Endpoint, :notice, activity) - redirect(conn, to: to) - - _ -> - not_found(conn, "Post not found.") - end - end - defp assign_id(%{path_info: ["notice", notice_id]} = conn, _opts), do: assign(conn, :notice_id, notice_id) diff --git a/test/support/conn_case.ex b/test/support/conn_case.ex index 7ef681258..7b28d70e7 100644 --- a/test/support/conn_case.ex +++ b/test/support/conn_case.ex @@ -111,28 +111,6 @@ defp json_response_and_validate_schema( defp json_response_and_validate_schema(conn, _status) do flunk("Response schema not found for #{conn.method} #{conn.request_path} #{conn.status}") end - - defp ensure_federating_or_authenticated(conn, url, user) do - initial_setting = Config.get([:instance, :federating]) - on_exit(fn -> Config.put([:instance, :federating], initial_setting) end) - - Config.put([:instance, :federating], false) - - conn - |> get(url) - |> response(403) - - conn - |> assign(:user, user) - |> get(url) - |> response(200) - - Config.put([:instance, :federating], true) - - conn - |> get(url) - |> response(200) - end end end diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 0517571f2..ab57b6523 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -33,6 +33,25 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do setup do: clear_config([:instance, :federating], true) + defp ensure_federating_or_authenticated(conn, url, user) do + Config.put([:instance, :federating], false) + + conn + |> get(url) + |> response(403) + + conn + |> assign(:user, user) + |> get(url) + |> response(200) + + Config.put([:instance, :federating], true) + + conn + |> get(url) + |> response(200) + end + describe "/relay" do setup do: clear_config([:instance, :allow_relay]) diff --git a/test/web/feed/user_controller_test.exs b/test/web/feed/user_controller_test.exs index 9a5610baa..7383e82cc 100644 --- a/test/web/feed/user_controller_test.exs +++ b/test/web/feed/user_controller_test.exs @@ -13,7 +13,7 @@ defmodule Pleroma.Web.Feed.UserControllerTest do alias Pleroma.User alias Pleroma.Web.CommonAPI - setup do: clear_config([:instance, :federating], true) + setup do: clear_config([:static_fe, :enabled], false) describe "feed" do setup do: clear_config([:feed]) @@ -192,6 +192,16 @@ test "returns 404 when the user is remote", %{conn: conn} do |> get(user_feed_path(conn, :feed, user.nickname)) |> response(404) end + + test "does not require authentication on non-federating instances", %{conn: conn} do + clear_config([:instance, :federating], false) + user = insert(:user) + + conn + |> put_req_header("accept", "application/rss+xml") + |> get("/users/#{user.nickname}/feed.rss") + |> response(200) + end end # Note: see ActivityPubControllerTest for JSON format tests diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index ee498f4b5..65b2c22db 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -7,7 +7,6 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do import Pleroma.Factory - alias Pleroma.Config alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub @@ -21,7 +20,7 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do :ok end - setup do: clear_config([:instance, :federating], true) + setup do: clear_config([:static_fe, :enabled], false) describe "Mastodon compatibility routes" do setup %{conn: conn} do @@ -215,15 +214,16 @@ test "404s a non-existing notice", %{conn: conn} do assert response(conn, 404) end - test "it requires authentication if instance is NOT federating", %{ + test "does not require authentication on non-federating instances", %{ conn: conn } do - user = insert(:user) + clear_config([:instance, :federating], false) note_activity = insert(:note_activity) - conn = put_req_header(conn, "accept", "text/html") - - ensure_federating_or_authenticated(conn, "/notice/#{note_activity.id}", user) + conn + |> put_req_header("accept", "text/html") + |> get("/notice/#{note_activity.id}") + |> response(200) end end @@ -325,14 +325,16 @@ test "404s when attachment isn't audio or video", %{conn: conn} do |> response(404) end - test "it requires authentication if instance is NOT federating", %{ + test "does not require authentication on non-federating instances", %{ conn: conn, note_activity: note_activity } do - user = insert(:user) - conn = put_req_header(conn, "accept", "text/html") + clear_config([:instance, :federating], false) - ensure_federating_or_authenticated(conn, "/notice/#{note_activity.id}/embed_player", user) + conn + |> put_req_header("accept", "text/html") + |> get("/notice/#{note_activity.id}/embed_player") + |> response(200) end end end diff --git a/test/web/static_fe/static_fe_controller_test.exs b/test/web/static_fe/static_fe_controller_test.exs index 1598bf675..bab0b0a7b 100644 --- a/test/web/static_fe/static_fe_controller_test.exs +++ b/test/web/static_fe/static_fe_controller_test.exs @@ -2,14 +2,12 @@ defmodule Pleroma.Web.StaticFE.StaticFEControllerTest do use Pleroma.Web.ConnCase alias Pleroma.Activity - alias Pleroma.Config alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.CommonAPI import Pleroma.Factory setup_all do: clear_config([:static_fe, :enabled], true) - setup do: clear_config([:instance, :federating], true) setup %{conn: conn} do conn = put_req_header(conn, "accept", "text/html") @@ -70,8 +68,15 @@ test "pagination, page 2", %{conn: conn, user: user} do refute html =~ ">test29<" end - test "it requires authentication if instance is NOT federating", %{conn: conn, user: user} do - ensure_federating_or_authenticated(conn, "/users/#{user.nickname}", user) + test "does not require authentication on non-federating instances", %{ + conn: conn, + user: user + } do + clear_config([:instance, :federating], false) + + conn = get(conn, "/users/#{user.nickname}") + + assert html_response(conn, 200) =~ user.nickname end end @@ -183,10 +188,17 @@ test "302 for remote cached status", %{conn: conn, user: user} do assert html_response(conn, 302) =~ "redirected" end - test "it requires authentication if instance is NOT federating", %{conn: conn, user: user} do + test "does not require authentication on non-federating instances", %{ + conn: conn, + user: user + } do + clear_config([:instance, :federating], false) + {:ok, activity} = CommonAPI.post(user, %{status: "testing a thing!"}) - ensure_federating_or_authenticated(conn, "/notice/#{activity.id}", user) + conn = get(conn, "/notice/#{activity.id}") + + assert html_response(conn, 200) =~ "testing a thing!" end end end From 094edde7c4ddf65f46e5d692a5ef5b859587d1e1 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 5 Oct 2020 23:48:00 +0300 Subject: [PATCH 05/75] [#3053] Unauthenticated access control for OStatus-related controllers and ActivityPubController (base actions: :user, :object, :activity). Tests adjustments. --- .../activity_pub/activity_pub_controller.ex | 56 +++++++++------- lib/pleroma/web/activity_pub/visibility.ex | 39 ++++++++--- lib/pleroma/web/feed/tag_controller.ex | 15 +++-- lib/pleroma/web/feed/user_controller.ex | 19 +++--- lib/pleroma/web/ostatus/ostatus_controller.ex | 26 +++---- lib/pleroma/web/router.ex | 52 ++++++++++---- .../web/static_fe/static_fe_controller.ex | 48 ++++++------- .../activity_pub_controller_test.exs | 67 ------------------- test/web/feed/tag_controller_test.exs | 13 ++-- 9 files changed, 159 insertions(+), 176 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 732c44271..c78edfb4c 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -32,17 +32,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do @federating_only_actions [:internal_fetch, :relay, :relay_following, :relay_followers] + # Note: :following and :followers must be served even without authentication (as via :api) + @auth_only_actions [:read_inbox, :update_outbox, :whoami, :upload_media] + + # Always accessible actions (must perform entity accessibility checks) + @no_auth_no_federation_actions [:user, :activity, :object] + + @authenticated_or_federating_actions @federating_only_actions ++ + @auth_only_actions ++ @no_auth_no_federation_actions + plug(FederatingPlug when action in @federating_only_actions) + plug(EnsureAuthenticatedPlug when action in @auth_only_actions) + plug( EnsureAuthenticatedPlug, - [unless_func: &FederatingPlug.federating?/1] when action not in @federating_only_actions - ) - - # Note: :following and :followers must be served even without authentication (as via :api) - plug( - EnsureAuthenticatedPlug - when action in [:read_inbox, :update_outbox, :whoami, :upload_media] + [unless_func: &FederatingPlug.federating?/1] + when action not in @authenticated_or_federating_actions ) plug( @@ -66,21 +72,22 @@ defp relay_active?(conn, _) do def user(conn, %{"nickname" => nickname}) do with %User{local: true} = user <- User.get_cached_by_nickname(nickname), + {_, :visible} <- {:visibility, User.visible_for(user, _reading_user = nil)}, {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_content_type("application/activity+json") |> put_view(UserView) |> render("user.json", %{user: user}) else - nil -> {:error, :not_found} - %{local: false} -> {:error, :not_found} + _ -> {:error, :not_found} end end def object(conn, _) do with ap_id <- Endpoint.url() <> conn.request_path, %Object{} = object <- Object.get_cached_by_ap_id(ap_id), - {_, true} <- {:public?, Visibility.is_public?(object)} do + {_, true} <- {:public?, Visibility.is_public?(object)}, + {_, false} <- {:restricted?, Visibility.restrict_unauthenticated_access?(object)} do conn |> assign(:tracking_fun_data, object.id) |> set_cache_ttl_for(object) @@ -88,25 +95,15 @@ def object(conn, _) do |> put_view(ObjectView) |> render("object.json", object: object) else - {:public?, false} -> - {:error, :not_found} + _ -> {:error, :not_found} end end - def track_object_fetch(conn, nil), do: conn - - def track_object_fetch(conn, object_id) do - with %{assigns: %{user: %User{id: user_id}}} <- conn do - Delivery.create(object_id, user_id) - end - - conn - end - def activity(conn, _params) do with ap_id <- Endpoint.url() <> conn.request_path, %Activity{} = activity <- Activity.normalize(ap_id), - {_, true} <- {:public?, Visibility.is_public?(activity)} do + {_, true} <- {:public?, Visibility.is_public?(activity)}, + {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)} do conn |> maybe_set_tracking_data(activity) |> set_cache_ttl_for(activity) @@ -114,8 +111,7 @@ def activity(conn, _params) do |> put_view(ObjectView) |> render("object.json", object: activity) else - {:public?, false} -> {:error, :not_found} - nil -> {:error, :not_found} + _ -> {:error, :not_found} end end @@ -550,4 +546,14 @@ def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = |> json(object.data) end end + + def track_object_fetch(conn, nil), do: conn + + def track_object_fetch(conn, object_id) do + with %{assigns: %{user: %User{id: user_id}}} <- conn do + Delivery.create(object_id, user_id) + end + + conn + end end diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex index 5c349bb7a..76bd54a42 100644 --- a/lib/pleroma/web/activity_pub/visibility.ex +++ b/lib/pleroma/web/activity_pub/visibility.ex @@ -44,29 +44,30 @@ def is_direct?(activity) do def is_list?(%{data: %{"listMessage" => _}}), do: true def is_list?(_), do: false - @spec visible_for_user?(Activity.t(), User.t() | nil) :: boolean() - def visible_for_user?(%{actor: ap_id}, %User{ap_id: ap_id}), do: true + @spec visible_for_user?(Activity.t() | nil, User.t() | nil) :: boolean() + def visible_for_user?(%Activity{actor: ap_id}, %User{ap_id: ap_id}), do: true def visible_for_user?(nil, _), do: false - def visible_for_user?(%{data: %{"listMessage" => _}}, nil), do: false + def visible_for_user?(%Activity{data: %{"listMessage" => _}}, nil), do: false - def visible_for_user?(%{data: %{"listMessage" => list_ap_id}} = activity, %User{} = user) do + def visible_for_user?( + %Activity{data: %{"listMessage" => list_ap_id}} = activity, + %User{} = user + ) do user.ap_id in activity.data["to"] || list_ap_id |> Pleroma.List.get_by_ap_id() |> Pleroma.List.member?(user) end - def visible_for_user?(%{local: local} = activity, nil) do - cfg_key = if local, do: :local, else: :remote - - if Pleroma.Config.restrict_unauthenticated_access?(:activities, cfg_key), + def visible_for_user?(%Activity{} = activity, nil) do + if restrict_unauthenticated_access?(activity), do: false, else: is_public?(activity) end - def visible_for_user?(activity, user) do + def visible_for_user?(%Activity{} = activity, user) do x = [user.ap_id | User.following(user)] y = [activity.actor] ++ activity.data["to"] ++ (activity.data["cc"] || []) is_public?(activity) || Enum.any?(x, &(&1 in y)) @@ -82,6 +83,26 @@ def entire_thread_visible_for_user?(%Activity{} = activity, %User{} = user) do result end + def restrict_unauthenticated_access?(%Activity{local: local}) do + restrict_unauthenticated_access_to_activity?(local) + end + + def restrict_unauthenticated_access?(%Object{} = object) do + object + |> Object.local?() + |> restrict_unauthenticated_access_to_activity?() + end + + def restrict_unauthenticated_access?(%User{} = user) do + User.visible_for(user, _reading_user = nil) + end + + defp restrict_unauthenticated_access_to_activity?(local?) when is_boolean(local?) do + cfg_key = if local?, do: :local, else: :remote + + Pleroma.Config.restrict_unauthenticated_access?(:activities, cfg_key) + end + def get_visibility(object) do to = object.data["to"] || [] cc = object.data["cc"] || [] diff --git a/lib/pleroma/web/feed/tag_controller.ex b/lib/pleroma/web/feed/tag_controller.ex index c348b32c2..218cdbdf3 100644 --- a/lib/pleroma/web/feed/tag_controller.ex +++ b/lib/pleroma/web/feed/tag_controller.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.Feed.TagController do alias Pleroma.Web.Feed.FeedView def feed(conn, params) do - unless Config.restrict_unauthenticated_access?(:activities, :local) do + if Config.get!([:instance, :public]) do render_feed(conn, params) else render_error(conn, :not_found, "Not found") @@ -36,12 +36,13 @@ defp render_feed(conn, %{"tag" => raw_tag} = params) do end @spec parse_tag(binary() | any()) :: {format :: String.t(), tag :: String.t()} - defp parse_tag(raw_tag) when is_binary(raw_tag) do - case Enum.reverse(String.split(raw_tag, ".")) do - [format | tag] when format in ["atom", "rss"] -> {format, Enum.join(tag, ".")} - _ -> {"rss", raw_tag} + defp parse_tag(raw_tag) do + case is_binary(raw_tag) && Enum.reverse(String.split(raw_tag, ".")) do + [format | tag] when format in ["rss", "atom"] -> + {format, Enum.join(tag, ".")} + + _ -> + {"atom", raw_tag} end end - - defp parse_tag(raw_tag), do: {"rss", raw_tag} end diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex index 5fbcd82d7..f1d2bb7be 100644 --- a/lib/pleroma/web/feed/user_controller.ex +++ b/lib/pleroma/web/feed/user_controller.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.Feed.UserController do use Pleroma.Web, :controller alias Fallback.RedirectController + + alias Pleroma.Config alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.ActivityPubController @@ -32,15 +34,7 @@ def feed_redirect(conn, %{"nickname" => nickname}) do end end - def feed(conn, params) do - unless Pleroma.Config.restrict_unauthenticated_access?(:profiles, :local) do - render_feed(conn, params) - else - errors(conn, {:error, :not_found}) - end - end - - defp render_feed(conn, %{"nickname" => nickname} = params) do + def feed(conn, %{"nickname" => nickname} = params) do format = get_format(conn) format = @@ -50,7 +44,8 @@ defp render_feed(conn, %{"nickname" => nickname} = params) do "atom" end - with {_, %User{local: true} = user} <- {:fetch_user, User.get_cached_by_nickname(nickname)} do + with {_, %User{local: true} = user} <- {:fetch_user, User.get_cached_by_nickname(nickname)}, + {_, :visible} <- {:visibility, User.visible_for(user, _reading_user = nil)} do activities = %{ type: ["Create"], @@ -65,7 +60,7 @@ defp render_feed(conn, %{"nickname" => nickname} = params) do |> render("user.#{format}", user: user, activities: activities, - feed_config: Pleroma.Config.get([:feed]) + feed_config: Config.get([:feed]) ) end end @@ -77,6 +72,8 @@ def errors(conn, {:error, :not_found}) do def errors(conn, {:fetch_user, %User{local: false}}), do: errors(conn, {:error, :not_found}) def errors(conn, {:fetch_user, nil}), do: errors(conn, {:error, :not_found}) + def errors(conn, {:visibility, _}), do: errors(conn, {:error, :not_found}) + def errors(conn, _) do render_error(conn, :internal_server_error, "Something went wrong") end diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 8646d2c1c..b4dc2a87f 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -33,16 +33,15 @@ def object(%{assigns: %{format: format}} = conn, _params) ActivityPubController.call(conn, :object) end - def object(%{assigns: %{format: format}} = conn, _params) do + def object(conn, _params) do with id <- Endpoint.url() <> conn.request_path, {_, %Activity{} = activity} <- {:activity, Activity.get_create_by_object_ap_id_with_object(id)}, - {_, true} <- {:public?, Visibility.is_public?(activity)} do - case format do - _ -> redirect(conn, to: "/notice/#{activity.id}") - end + {_, true} <- {:public?, Visibility.is_public?(activity)}, + {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)} do + redirect(conn, to: "/notice/#{activity.id}") else - reason when reason in [{:public?, false}, {:activity, nil}] -> + reason when reason in [{:public?, false}, {:visible?, false}, {:activity, nil}] -> {:error, :not_found} e -> @@ -55,15 +54,14 @@ def activity(%{assigns: %{format: format}} = conn, _params) ActivityPubController.call(conn, :activity) end - def activity(%{assigns: %{format: format}} = conn, _params) do + def activity(conn, _params) do with id <- Endpoint.url() <> conn.request_path, {_, %Activity{} = activity} <- {:activity, Activity.normalize(id)}, - {_, true} <- {:public?, Visibility.is_public?(activity)} do - case format do - _ -> redirect(conn, to: "/notice/#{activity.id}") - end + {_, true} <- {:public?, Visibility.is_public?(activity)}, + {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)} do + redirect(conn, to: "/notice/#{activity.id}") else - reason when reason in [{:public?, false}, {:activity, nil}] -> + reason when reason in [{:public?, false}, {:visible?, false}, {:activity, nil}] -> {:error, :not_found} e -> @@ -74,6 +72,7 @@ def activity(%{assigns: %{format: format}} = conn, _params) do def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id_with_object(id)}, {_, true} <- {:public?, Visibility.is_public?(activity)}, + {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)}, %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do cond do format in ["json", "activity+json"] -> @@ -101,7 +100,7 @@ def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do RedirectController.redirector(conn, nil) end else - reason when reason in [{:public?, false}, {:activity, nil}] -> + reason when reason in [{:public?, false}, {:visible?, false}, {:activity, nil}] -> conn |> put_status(404) |> RedirectController.redirector(nil, 404) @@ -115,6 +114,7 @@ def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do def notice_player(conn, %{"id" => id}) do with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id_with_object(id), true <- Visibility.is_public?(activity), + {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)}, %Object{} = object <- Object.normalize(activity), %{data: %{"attachment" => [%{"url" => [url | _]} | _]}} <- object, true <- String.starts_with?(url["mediaType"], ["audio", "video"]) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e0e92549f..6439a1c39 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -5,6 +5,14 @@ defmodule Pleroma.Web.Router do use Pleroma.Web, :router + pipeline :accepts_html do + plug(:accepts, ["html"]) + end + + pipeline :accepts_xml_rss_atom do + plug(:accepts, ["xml", "rss", "atom"]) + end + pipeline :browser do plug(:accepts, ["html"]) plug(:fetch_session) @@ -556,39 +564,55 @@ defmodule Pleroma.Web.Router do ) end - pipeline :ostatus do + pipeline :ostatus_html_json do + plug(:accepts, ["html", "activity+json", "json"]) + plug(Pleroma.Plugs.StaticFEPlug) + end + + pipeline :ostatus_html_xml do + plug(:accepts, ["html", "xml", "rss", "atom"]) + plug(Pleroma.Plugs.StaticFEPlug) + end + + pipeline :ostatus_html_xml_json do plug(:accepts, ["html", "xml", "rss", "atom", "activity+json", "json"]) plug(Pleroma.Plugs.StaticFEPlug) end - pipeline :ostatus_no_html do - plug(:accepts, ["xml", "rss", "atom", "activity+json", "json"]) - end - - pipeline :oembed do - plug(:accepts, ["json", "xml"]) - end - scope "/", Pleroma.Web do - # Note: no authentication plugs, all endpoints below should only yield public objects - pipe_through(:ostatus) + # Note: html format is supported only if static FE is enabled + pipe_through(:ostatus_html_json) get("/objects/:uuid", OStatus.OStatusController, :object) get("/activities/:uuid", OStatus.OStatusController, :activity) get("/notice/:id", OStatus.OStatusController, :notice) - get("/notice/:id/embed_player", OStatus.OStatusController, :notice_player) # Mastodon compatibility routes get("/users/:nickname/statuses/:id", OStatus.OStatusController, :object) get("/users/:nickname/statuses/:id/activity", OStatus.OStatusController, :activity) + end - get("/users/:nickname/feed", Feed.UserController, :feed, as: :user_feed) + scope "/", Pleroma.Web do + # Note: html format is supported only if static FE is enabled + pipe_through(:ostatus_html_xml_json) + + # Note: for json format responds with user profile (not user feed) get("/users/:nickname", Feed.UserController, :feed_redirect, as: :user_feed) end scope "/", Pleroma.Web do - pipe_through(:ostatus_no_html) + # Note: html format is supported only if static FE is enabled + pipe_through(:ostatus_html_xml) + get("/users/:nickname/feed", Feed.UserController, :feed, as: :user_feed) + end + scope "/", Pleroma.Web do + pipe_through(:accepts_html) + get("/notice/:id/embed_player", OStatus.OStatusController, :notice_player) + end + + scope "/", Pleroma.Web do + pipe_through(:accepts_xml_rss_atom) get("/tags/:tag", Feed.TagController, :feed, as: :tag_feed) end diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index b1c62f5b0..76b82589f 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -24,6 +24,7 @@ def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do with %Activity{local: true} = activity <- Activity.get_by_id_with_object(notice_id), true <- Visibility.is_public?(activity.object), + {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)}, %User{} = user <- User.get_by_ap_id(activity.object.data["actor"]) do meta = Metadata.build_tags(%{activity_id: notice_id, object: activity.object, user: user}) @@ -47,34 +48,35 @@ def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do @doc "Renders public activities of requested user" def show(%{assigns: %{username_or_id: username_or_id}} = conn, params) do - case User.get_cached_by_nickname_or_id(username_or_id) do - %User{} = user -> - meta = Metadata.build_tags(%{user: user}) + with {_, %User{local: true} = user} <- + {:fetch_user, User.get_cached_by_nickname_or_id(username_or_id)}, + {_, :visible} <- {:visibility, User.visible_for(user, _reading_user = nil)} do + meta = Metadata.build_tags(%{user: user}) - params = - params - |> Map.take(@page_keys) - |> Map.new(fn {k, v} -> {String.to_existing_atom(k), v} end) + params = + params + |> Map.take(@page_keys) + |> Map.new(fn {k, v} -> {String.to_existing_atom(k), v} end) - timeline = - user - |> ActivityPub.fetch_user_activities(_reading_user = nil, params) - |> Enum.map(&represent/1) + timeline = + user + |> ActivityPub.fetch_user_activities(_reading_user = nil, params) + |> Enum.map(&represent/1) - prev_page_id = - (params["min_id"] || params["max_id"]) && - List.first(timeline) && List.first(timeline).id + prev_page_id = + (params["min_id"] || params["max_id"]) && + List.first(timeline) && List.first(timeline).id - next_page_id = List.last(timeline) && List.last(timeline).id - - render(conn, "profile.html", %{ - user: User.sanitize_html(user), - timeline: timeline, - prev_page_id: prev_page_id, - next_page_id: next_page_id, - meta: meta - }) + next_page_id = List.last(timeline) && List.last(timeline).id + render(conn, "profile.html", %{ + user: User.sanitize_html(user), + timeline: timeline, + prev_page_id: prev_page_id, + next_page_id: next_page_id, + meta: meta + }) + else _ -> not_found(conn, "User not found.") end diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index ab57b6523..9ec13b21f 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -33,25 +33,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do setup do: clear_config([:instance, :federating], true) - defp ensure_federating_or_authenticated(conn, url, user) do - Config.put([:instance, :federating], false) - - conn - |> get(url) - |> response(403) - - conn - |> assign(:user, user) - |> get(url) - |> response(200) - - Config.put([:instance, :federating], true) - - conn - |> get(url) - |> response(200) - end - describe "/relay" do setup do: clear_config([:instance, :allow_relay]) @@ -175,21 +156,6 @@ test "it returns error when user is not found", %{conn: conn} do assert response == "Not found" end - - test "it requires authentication if instance is NOT federating", %{ - conn: conn - } do - user = insert(:user) - - conn = - put_req_header( - conn, - "accept", - "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"" - ) - - ensure_federating_or_authenticated(conn, "/users/#{user.nickname}.json", user) - end end describe "mastodon compatibility routes" do @@ -357,18 +323,6 @@ test "cached purged after object deletion", %{conn: conn} do assert "Not found" == json_response(conn2, :not_found) end - - test "it requires authentication if instance is NOT federating", %{ - conn: conn - } do - user = insert(:user) - note = insert(:note) - uuid = String.split(note.data["id"], "/") |> List.last() - - conn = put_req_header(conn, "accept", "application/activity+json") - - ensure_federating_or_authenticated(conn, "/objects/#{uuid}", user) - end end describe "/activities/:uuid" do @@ -440,18 +394,6 @@ test "cached purged after activity deletion", %{conn: conn} do assert "Not found" == json_response(conn2, :not_found) end - - test "it requires authentication if instance is NOT federating", %{ - conn: conn - } do - user = insert(:user) - activity = insert(:note_activity) - uuid = String.split(activity.data["id"], "/") |> List.last() - - conn = put_req_header(conn, "accept", "application/activity+json") - - ensure_federating_or_authenticated(conn, "/activities/#{uuid}", user) - end end describe "/inbox" do @@ -912,15 +854,6 @@ test "it returns an announce activity in a collection", %{conn: conn} do assert response(conn, 200) =~ announce_activity.data["object"] end - - test "it requires authentication if instance is NOT federating", %{ - conn: conn - } do - user = insert(:user) - conn = put_req_header(conn, "accept", "application/activity+json") - - ensure_federating_or_authenticated(conn, "/users/#{user.nickname}/outbox", user) - end end describe "POST /users/:nickname/outbox (C2S)" do diff --git a/test/web/feed/tag_controller_test.exs b/test/web/feed/tag_controller_test.exs index 868e40965..e4084b0e5 100644 --- a/test/web/feed/tag_controller_test.exs +++ b/test/web/feed/tag_controller_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.Web.Feed.TagControllerTest do import Pleroma.Factory import SweetXml + alias Pleroma.Config alias Pleroma.Object alias Pleroma.Web.CommonAPI alias Pleroma.Web.Feed.FeedView @@ -15,7 +16,7 @@ defmodule Pleroma.Web.Feed.TagControllerTest do setup do: clear_config([:feed]) test "gets a feed (ATOM)", %{conn: conn} do - Pleroma.Config.put( + Config.put( [:feed, :post_title], %{max_length: 25, omission: "..."} ) @@ -82,7 +83,7 @@ test "gets a feed (ATOM)", %{conn: conn} do end test "gets a feed (RSS)", %{conn: conn} do - Pleroma.Config.put( + Config.put( [:feed, :post_title], %{max_length: 25, omission: "..."} ) @@ -157,7 +158,7 @@ test "gets a feed (RSS)", %{conn: conn} do response = conn |> put_req_header("accept", "application/rss+xml") - |> get(tag_feed_path(conn, :feed, "pleromaart")) + |> get(tag_feed_path(conn, :feed, "pleromaart.rss")) |> response(200) xml = parse(response) @@ -183,14 +184,12 @@ test "gets a feed (RSS)", %{conn: conn} do end describe "private instance" do - setup do: clear_config([:instance, :public]) + setup do: clear_config([:instance, :public], false) test "returns 404 for tags feed", %{conn: conn} do - Config.put([:instance, :public], false) - conn |> put_req_header("accept", "application/rss+xml") - |> get(tag_feed_path(conn, :feed, "pleromaart")) + |> get(tag_feed_path(conn, :feed, "pleromaart.rss")) |> response(404) end end From 257e059e61b89752bcde9544cb5ae645b167c96b Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 19 Aug 2020 15:31:33 +0400 Subject: [PATCH 06/75] Add account export --- lib/pleroma/export.ex | 118 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 lib/pleroma/export.ex diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex new file mode 100644 index 000000000..82a4b7ace --- /dev/null +++ b/lib/pleroma/export.ex @@ -0,0 +1,118 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Export do + alias Pleroma.Activity + alias Pleroma.Bookmark + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.ActivityPub.UserView + + import Ecto.Query + + def run(user) do + with {:ok, dir} <- create_dir(), + :ok <- actor(dir, user), + :ok <- statuses(dir, user), + :ok <- likes(dir, user), + :ok <- bookmarks(dir, user) do + IO.inspect({"DONE", dir}) + else + err -> IO.inspect({"export error", err}) + end + end + + def actor(dir, user) do + with {:ok, json} <- + UserView.render("user.json", %{user: user}) + |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"}) + |> Jason.encode() do + File.write(dir <> "/actor.json", json) + end + end + + defp create_dir do + datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now()) + dir = Path.join(System.tmp_dir!(), "archive-" <> datetime) + + with :ok <- File.mkdir(dir), do: {:ok, dir} + end + + defp write_header(file, name) do + IO.write( + file, + """ + { + "@context": "https://www.w3.org/ns/activitystreams", + "id": "#{name}.json", + "type": "OrderedCollection", + "orderedItems": [ + """ + ) + end + + defp write(query, dir, name, fun) do + path = dir <> "/#{name}.json" + + with {:ok, file} <- File.open(path, [:write, :utf8]), + :ok <- write_header(file, name) do + counter = :counters.new(1, []) + + query + |> Pleroma.RepoStreamer.chunk_stream(100) + |> Stream.each(fn items -> + Enum.each(items, fn i -> + with {:ok, str} <- fun.(i), + :ok <- IO.write(file, str <> ",\n") do + :counters.add(counter, 1, 1) + end + end) + end) + |> Stream.run() + + total = :counters.get(counter, 1) + + with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n \"totalItems\": #{total}}") do + File.close(file) + end + end + end + + def bookmarks(dir, %{id: user_id} = _user) do + Bookmark + |> where(user_id: ^user_id) + |> join(:inner, [b], activity in assoc(b, :activity)) + |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)}) + |> write(dir, "bookmarks", fn a -> {:ok, "\"#{a.object}\""} end) + end + + def likes(dir, user) do + user.ap_id + |> Activity.Queries.by_actor() + |> Activity.Queries.by_type("Like") + |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)}) + |> write(dir, "likes", fn a -> {:ok, "\"#{a.object}\""} end) + end + + def statuses(dir, user) do + opts = + %{} + |> Map.put(:type, ["Create", "Announce"]) + |> Map.put(:blocking_user, user) + |> Map.put(:muting_user, user) + |> Map.put(:reply_filtering_user, user) + |> Map.put(:announce_filtering_user, user) + |> Map.put(:user, user) + + [[user.ap_id], User.following(user), Pleroma.List.memberships(user)] + |> Enum.concat() + |> ActivityPub.fetch_activities_query(opts) + |> write(dir, "outbox", fn a -> + with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do + activity |> Map.delete("@context") |> Jason.encode() + end + end) + end +end From 9d564ffc2988f145bc9cf26477eea93b1bf01cb0 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 24 Aug 2020 20:59:57 +0400 Subject: [PATCH 07/75] Zip exported files --- lib/pleroma/export.ex | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex index 82a4b7ace..f0f1ef093 100644 --- a/lib/pleroma/export.ex +++ b/lib/pleroma/export.ex @@ -12,15 +12,17 @@ defmodule Pleroma.Export do import Ecto.Query + @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json'] + def run(user) do - with {:ok, dir} <- create_dir(), - :ok <- actor(dir, user), - :ok <- statuses(dir, user), - :ok <- likes(dir, user), - :ok <- bookmarks(dir, user) do - IO.inspect({"DONE", dir}) - else - err -> IO.inspect({"export error", err}) + with {:ok, path} <- create_dir(user), + :ok <- actor(path, user), + :ok <- statuses(path, user), + :ok <- likes(path, user), + :ok <- bookmarks(path, user), + {:ok, zip_path} <- :zip.create('#{path}.zip', @files, cwd: path), + {:ok, _} <- File.rm_rf(path) do + {:ok, zip_path} end end @@ -33,9 +35,9 @@ def actor(dir, user) do end end - defp create_dir do + defp create_dir(user) do datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now()) - dir = Path.join(System.tmp_dir!(), "archive-" <> datetime) + dir = Path.join(System.tmp_dir!(), "archive-#{user.id}-#{datetime}") with :ok <- File.mkdir(dir), do: {:ok, dir} end From c01a81804835fb92c145b90e3a264c5d4cf9c886 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 25 Aug 2020 18:51:09 +0400 Subject: [PATCH 08/75] Add tests --- lib/pleroma/export.ex | 8 +-- test/export_test.exs | 111 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 4 deletions(-) create mode 100644 test/export_test.exs diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex index f0f1ef093..45b8ce749 100644 --- a/lib/pleroma/export.ex +++ b/lib/pleroma/export.ex @@ -26,7 +26,7 @@ def run(user) do end end - def actor(dir, user) do + defp actor(dir, user) do with {:ok, json} <- UserView.render("user.json", %{user: user}) |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"}) @@ -82,7 +82,7 @@ defp write(query, dir, name, fun) do end end - def bookmarks(dir, %{id: user_id} = _user) do + defp bookmarks(dir, %{id: user_id} = _user) do Bookmark |> where(user_id: ^user_id) |> join(:inner, [b], activity in assoc(b, :activity)) @@ -90,7 +90,7 @@ def bookmarks(dir, %{id: user_id} = _user) do |> write(dir, "bookmarks", fn a -> {:ok, "\"#{a.object}\""} end) end - def likes(dir, user) do + defp likes(dir, user) do user.ap_id |> Activity.Queries.by_actor() |> Activity.Queries.by_type("Like") @@ -98,7 +98,7 @@ def likes(dir, user) do |> write(dir, "likes", fn a -> {:ok, "\"#{a.object}\""} end) end - def statuses(dir, user) do + defp statuses(dir, user) do opts = %{} |> Map.put(:type, ["Create", "Announce"]) diff --git a/test/export_test.exs b/test/export_test.exs new file mode 100644 index 000000000..5afd58ccc --- /dev/null +++ b/test/export_test.exs @@ -0,0 +1,111 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ExportTest do + use Pleroma.DataCase + import Pleroma.Factory + + alias Pleroma.Web.CommonAPI + alias Pleroma.Bookmark + + test "it exports user data" do + user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) + + {:ok, %{object: %{data: %{"id" => id1}}} = status1} = + CommonAPI.post(user, %{status: "status1"}) + + {:ok, %{object: %{data: %{"id" => id2}}} = status2} = + CommonAPI.post(user, %{status: "status2"}) + + {:ok, %{object: %{data: %{"id" => id3}}} = status3} = + CommonAPI.post(user, %{status: "status3"}) + + CommonAPI.favorite(user, status1.id) + CommonAPI.favorite(user, status2.id) + + Bookmark.create(user.id, status2.id) + Bookmark.create(user.id, status3.id) + + assert {:ok, path} = Pleroma.Export.run(user) + assert {:ok, zipfile} = :zip.zip_open(path, [:memory]) + assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile) + + assert %{ + "@context" => [ + "https://www.w3.org/ns/activitystreams", + "http://localhost:4001/schemas/litepub-0.1.jsonld", + %{"@language" => "und"} + ], + "bookmarks" => "bookmarks.json", + "followers" => "http://cofe.io/users/cofe/followers", + "following" => "http://cofe.io/users/cofe/following", + "id" => "http://cofe.io/users/cofe", + "inbox" => "http://cofe.io/users/cofe/inbox", + "likes" => "likes.json", + "name" => "Cofe", + "outbox" => "http://cofe.io/users/cofe/outbox", + "preferredUsername" => "cofe", + "publicKey" => %{ + "id" => "http://cofe.io/users/cofe#main-key", + "owner" => "http://cofe.io/users/cofe" + }, + "type" => "Person", + "url" => "http://cofe.io/users/cofe" + } = Jason.decode!(json) + + assert {:ok, {'outbox.json', json}} = :zip.zip_get('outbox.json', zipfile) + + assert %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "id" => "outbox.json", + "orderedItems" => [ + %{ + "object" => %{ + "actor" => "http://cofe.io/users/cofe", + "content" => "status1", + "type" => "Note" + }, + "type" => "Create" + }, + %{ + "object" => %{ + "actor" => "http://cofe.io/users/cofe", + "content" => "status2" + } + }, + %{ + "actor" => "http://cofe.io/users/cofe", + "object" => %{ + "content" => "status3" + } + } + ], + "totalItems" => 3, + "type" => "OrderedCollection" + } = Jason.decode!(json) + + assert {:ok, {'likes.json', json}} = :zip.zip_get('likes.json', zipfile) + + assert %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "id" => "likes.json", + "orderedItems" => [^id1, ^id2], + "totalItems" => 2, + "type" => "OrderedCollection" + } = Jason.decode!(json) + + assert {:ok, {'bookmarks.json', json}} = :zip.zip_get('bookmarks.json', zipfile) + + assert %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "id" => "bookmarks.json", + "orderedItems" => [^id2, ^id3], + "totalItems" => 2, + "type" => "OrderedCollection" + } = Jason.decode!(json) + + :zip.zip_close(zipfile) + File.rm!(path) + end +end From c82f9129592553718be4bd4712a2b1848dd0a447 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 25 Aug 2020 19:16:01 +0400 Subject: [PATCH 09/75] Fix credo warning --- test/export_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/export_test.exs b/test/export_test.exs index 5afd58ccc..01ca8e7e8 100644 --- a/test/export_test.exs +++ b/test/export_test.exs @@ -6,8 +6,8 @@ defmodule Pleroma.ExportTest do use Pleroma.DataCase import Pleroma.Factory - alias Pleroma.Web.CommonAPI alias Pleroma.Bookmark + alias Pleroma.Web.CommonAPI test "it exports user data" do user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) From be42ab70dc9538df54ac6f30ee123666223b7287 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 31 Aug 2020 20:31:21 +0400 Subject: [PATCH 10/75] Add backup upload --- lib/pleroma/export.ex | 20 +++++++++++++++++++- test/export_test.exs | 17 ++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex index 45b8ce749..b84eccd78 100644 --- a/lib/pleroma/export.ex +++ b/lib/pleroma/export.ex @@ -22,7 +22,25 @@ def run(user) do :ok <- bookmarks(path, user), {:ok, zip_path} <- :zip.create('#{path}.zip', @files, cwd: path), {:ok, _} <- File.rm_rf(path) do - {:ok, zip_path} + {:ok, :binary.list_to_bin(zip_path)} + end + end + + def upload(zip_path) do + uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) + file_name = zip_path |> String.split("/") |> List.last() + id = Ecto.UUID.generate() + + upload = %Pleroma.Upload{ + id: id, + name: file_name, + tempfile: zip_path, + content_type: "application/zip", + path: id <> "/" <> file_name + } + + with :ok <- uploader.put_file(upload), :ok <- File.rm(zip_path) do + {:ok, upload} end end diff --git a/test/export_test.exs b/test/export_test.exs index 01ca8e7e8..fae269974 100644 --- a/test/export_test.exs +++ b/test/export_test.exs @@ -28,7 +28,7 @@ test "it exports user data" do Bookmark.create(user.id, status3.id) assert {:ok, path} = Pleroma.Export.run(user) - assert {:ok, zipfile} = :zip.zip_open(path, [:memory]) + assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory]) assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile) assert %{ @@ -108,4 +108,19 @@ test "it exports user data" do :zip.zip_close(zipfile) File.rm!(path) end + + test "it uploads an exported backup archive" do + user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) + + {:ok, status1} = CommonAPI.post(user, %{status: "status1"}) + {:ok, status2} = CommonAPI.post(user, %{status: "status2"}) + {:ok, status3} = CommonAPI.post(user, %{status: "status3"}) + CommonAPI.favorite(user, status1.id) + CommonAPI.favorite(user, status2.id) + Bookmark.create(user.id, status2.id) + Bookmark.create(user.id, status3.id) + + assert {:ok, path} = Pleroma.Export.run(user) + assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) + end end From 75e07ba206b94155c5210151a49e29a11bce6e50 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 31 Aug 2020 23:07:14 +0400 Subject: [PATCH 11/75] Fix tests --- lib/pleroma/export.ex | 3 ++- test/export_test.exs | 47 +++++++++++++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex index b84eccd78..8b1bfefe2 100644 --- a/lib/pleroma/export.ex +++ b/lib/pleroma/export.ex @@ -39,7 +39,8 @@ def upload(zip_path) do path: id <> "/" <> file_name } - with :ok <- uploader.put_file(upload), :ok <- File.rm(zip_path) do + with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload), + :ok <- File.rm(zip_path) do {:ok, upload} end end diff --git a/test/export_test.exs b/test/export_test.exs index fae269974..d7e8f558c 100644 --- a/test/export_test.exs +++ b/test/export_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.ExportTest do use Pleroma.DataCase import Pleroma.Factory + import Mock alias Pleroma.Bookmark alias Pleroma.Web.CommonAPI @@ -109,18 +110,42 @@ test "it exports user data" do File.rm!(path) end - test "it uploads an exported backup archive" do - user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) + describe "it uploads an exported backup archive" do + setup do + clear_config(Pleroma.Uploaders.S3, + bucket: "test_bucket", + public_endpoint: "https://s3.amazonaws.com" + ) - {:ok, status1} = CommonAPI.post(user, %{status: "status1"}) - {:ok, status2} = CommonAPI.post(user, %{status: "status2"}) - {:ok, status3} = CommonAPI.post(user, %{status: "status3"}) - CommonAPI.favorite(user, status1.id) - CommonAPI.favorite(user, status2.id) - Bookmark.create(user.id, status2.id) - Bookmark.create(user.id, status3.id) + clear_config([Pleroma.Upload, :uploader]) - assert {:ok, path} = Pleroma.Export.run(user) - assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) + user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) + + {:ok, status1} = CommonAPI.post(user, %{status: "status1"}) + {:ok, status2} = CommonAPI.post(user, %{status: "status2"}) + {:ok, status3} = CommonAPI.post(user, %{status: "status3"}) + CommonAPI.favorite(user, status1.id) + CommonAPI.favorite(user, status2.id) + Bookmark.create(user.id, status2.id) + Bookmark.create(user.id, status3.id) + + assert {:ok, path} = Pleroma.Export.run(user) + + [path: path] + end + + test "S3", %{path: path} do + Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3) + + with_mock ExAws, request: fn _ -> {:ok, :ok} end do + assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) + end + end + + test "Local", %{path: path} do + Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) + + assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) + end end end From 4f3a6337454807f4145bbc1830c3d55dd883d46d Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 2 Sep 2020 20:21:33 +0400 Subject: [PATCH 12/75] Add `backups` table --- lib/pleroma/{export.ex => backup.ex} | 110 ++++++++++++++---- .../20200831192323_create_backups.exs | 17 +++ test/{export_test.exs => backup_test.exs} | 48 ++++++-- 3 files changed, 141 insertions(+), 34 deletions(-) rename lib/pleroma/{export.ex => backup.ex} (60%) create mode 100644 priv/repo/migrations/20200831192323_create_backups.exs rename test/{export_test.exs => backup_test.exs} (75%) diff --git a/lib/pleroma/export.ex b/lib/pleroma/backup.ex similarity index 60% rename from lib/pleroma/export.ex rename to lib/pleroma/backup.ex index 8b1bfefe2..4580d8f92 100644 --- a/lib/pleroma/export.ex +++ b/lib/pleroma/backup.ex @@ -2,41 +2,110 @@ # Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Export do +defmodule Pleroma.Backup do + use Ecto.Schema + + import Ecto.Changeset + import Ecto.Query + alias Pleroma.Activity alias Pleroma.Bookmark + alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.UserView - import Ecto.Query + schema "backups" do + field(:content_type, :string) + field(:file_name, :string) + field(:file_size, :integer, default: 0) + field(:processed, :boolean, default: false) + + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) + + timestamps() + end + + def create(user) do + with :ok <- validate_limit(user), + {:ok, backup} <- user |> new() |> Repo.insert() do + {:ok, backup} + end + end + + def new(user) do + rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false) + datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now()) + name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip" + + %__MODULE__{ + user_id: user.id, + content_type: "application/zip", + file_name: name + } + end + + defp validate_limit(user) do + case get_last(user.id) do + %__MODULE__{inserted_at: inserted_at} -> + days = 7 + diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days) + + if diff > days do + :ok + else + {:error, "Last export was less than #{days} days ago"} + end + + nil -> + :ok + end + end + + def get_last(user_id) do + __MODULE__ + |> where(user_id: ^user_id) + |> order_by(desc: :id) + |> limit(1) + |> Repo.one() + end + + def process(%__MODULE__{} = backup) do + with {:ok, zip_file} <- zip(backup), + {:ok, %{size: size}} <- File.stat(zip_file), + {:ok, _upload} <- upload(backup, zip_file) do + backup + |> cast(%{file_size: size, processed: true}, [:file_size, :processed]) + |> Repo.update() + end + end @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json'] + def zip(%__MODULE__{} = backup) do + backup = Repo.preload(backup, :user) + name = String.trim_trailing(backup.file_name, ".zip") + dir = Path.join(System.tmp_dir!(), name) - def run(user) do - with {:ok, path} <- create_dir(user), - :ok <- actor(path, user), - :ok <- statuses(path, user), - :ok <- likes(path, user), - :ok <- bookmarks(path, user), - {:ok, zip_path} <- :zip.create('#{path}.zip', @files, cwd: path), - {:ok, _} <- File.rm_rf(path) do + with :ok <- File.mkdir(dir), + :ok <- actor(dir, backup.user), + :ok <- statuses(dir, backup.user), + :ok <- likes(dir, backup.user), + :ok <- bookmarks(dir, backup.user), + {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir), + {:ok, _} <- File.rm_rf(dir) do {:ok, :binary.list_to_bin(zip_path)} end end - def upload(zip_path) do + def upload(%__MODULE__{} = backup, zip_path) do uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) - file_name = zip_path |> String.split("/") |> List.last() - id = Ecto.UUID.generate() upload = %Pleroma.Upload{ - id: id, - name: file_name, + name: backup.file_name, tempfile: zip_path, - content_type: "application/zip", - path: id <> "/" <> file_name + content_type: backup.content_type, + path: "backups/" <> backup.file_name } with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload), @@ -54,13 +123,6 @@ defp actor(dir, user) do end end - defp create_dir(user) do - datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now()) - dir = Path.join(System.tmp_dir!(), "archive-#{user.id}-#{datetime}") - - with :ok <- File.mkdir(dir), do: {:ok, dir} - end - defp write_header(file, name) do IO.write( file, diff --git a/priv/repo/migrations/20200831192323_create_backups.exs b/priv/repo/migrations/20200831192323_create_backups.exs new file mode 100644 index 000000000..3ac5889e2 --- /dev/null +++ b/priv/repo/migrations/20200831192323_create_backups.exs @@ -0,0 +1,17 @@ +defmodule Pleroma.Repo.Migrations.CreateBackups do + use Ecto.Migration + + def change do + create_if_not_exists table(:backups) do + add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) + add(:file_name, :string, null: false) + add(:content_type, :string, null: false) + add(:processed, :boolean, null: false, default: false) + add(:file_size, :bigint) + + timestamps() + end + + create_if_not_exists(index(:backups, [:user_id])) + end +end diff --git a/test/export_test.exs b/test/backup_test.exs similarity index 75% rename from test/export_test.exs rename to test/backup_test.exs index d7e8f558c..27f5cb7f7 100644 --- a/test/export_test.exs +++ b/test/backup_test.exs @@ -2,15 +2,41 @@ # Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.ExportTest do +defmodule Pleroma.BackupTest do use Pleroma.DataCase import Pleroma.Factory import Mock + alias Pleroma.Backup alias Pleroma.Bookmark alias Pleroma.Web.CommonAPI - test "it exports user data" do + test "it creates a backup record" do + %{id: user_id} = user = insert(:user) + assert {:ok, backup} = Backup.create(user) + + assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup + end + + test "it return an error if the export limit is over" do + %{id: user_id} = user = insert(:user) + limit_days = 7 + + assert {:ok, backup} = Backup.create(user) + assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup + + assert Backup.create(user) == {:error, "Last export was less than #{limit_days} days ago"} + end + + test "it process a backup record" do + %{id: user_id} = user = insert(:user) + assert {:ok, %{id: backup_id} = backup} = Backup.create(user) + assert {:ok, %Backup{} = backup} = Backup.process(backup) + assert backup.file_size > 0 + assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup + end + + test "it creates a zip archive with user data" do user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) {:ok, %{object: %{data: %{"id" => id1}}} = status1} = @@ -28,7 +54,8 @@ test "it exports user data" do Bookmark.create(user.id, status2.id) Bookmark.create(user.id, status3.id) - assert {:ok, path} = Pleroma.Export.run(user) + assert {:ok, backup} = user |> Backup.new() |> Repo.insert() + assert {:ok, path} = Backup.zip(backup) assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory]) assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile) @@ -110,7 +137,7 @@ test "it exports user data" do File.rm!(path) end - describe "it uploads an exported backup archive" do + describe "it uploads a backup archive" do setup do clear_config(Pleroma.Uploaders.S3, bucket: "test_bucket", @@ -129,23 +156,24 @@ test "it exports user data" do Bookmark.create(user.id, status2.id) Bookmark.create(user.id, status3.id) - assert {:ok, path} = Pleroma.Export.run(user) + assert {:ok, backup} = user |> Backup.new() |> Repo.insert() + assert {:ok, path} = Backup.zip(backup) - [path: path] + [path: path, backup: backup] end - test "S3", %{path: path} do + test "S3", %{path: path, backup: backup} do Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3) with_mock ExAws, request: fn _ -> {:ok, :ok} end do - assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) + assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path) end end - test "Local", %{path: path} do + test "Local", %{path: path, backup: backup} do Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) - assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) + assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path) end end end From a0ad9bd734e9af0ce912c32c7480a60ff87a4368 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 2 Sep 2020 21:45:22 +0400 Subject: [PATCH 13/75] Add BackupWorker --- config/config.exs | 1 + config/description.exs | 6 ++++++ lib/pleroma/backup.ex | 11 ++++++++++- lib/pleroma/workers/backup_worker.ex | 17 +++++++++++++++++ test/backup_test.exs | 20 ++++++++++++++------ 5 files changed, 48 insertions(+), 7 deletions(-) create mode 100644 lib/pleroma/workers/backup_worker.ex diff --git a/config/config.exs b/config/config.exs index 2e6b0796a..1f10167e5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -551,6 +551,7 @@ queues: [ activity_expiration: 10, token_expiration: 5, + backup: 1, federator_incoming: 50, federator_outgoing: 50, ingestion_queue: 50, diff --git a/config/description.exs b/config/description.exs index 6fa78a5d1..13e44afe8 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2288,6 +2288,12 @@ description: "Activity expiration queue", suggestions: [10] }, + %{ + key: :backup, + type: :integer, + description: "Backup queue", + suggestions: [1] + }, %{ key: :attachments_cleanup, type: :integer, diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index 4580d8f92..9b5d2625f 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -30,7 +30,7 @@ defmodule Pleroma.Backup do def create(user) do with :ok <- validate_limit(user), {:ok, backup} <- user |> new() |> Repo.insert() do - {:ok, backup} + Pleroma.Workers.BackupWorker.enqueue("process", %{"backup_id" => backup.id}) end end @@ -71,6 +71,15 @@ def get_last(user_id) do |> Repo.one() end + def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do + __MODULE__ + |> where(user_id: ^user_id) + |> where([b], b.id != ^latest_id) + |> Repo.delete_all() + end + + def get(id), do: Repo.get(__MODULE__, id) + def process(%__MODULE__{} = backup) do with {:ok, zip_file} <- zip(backup), {:ok, %{size: size}} <- File.stat(zip_file), diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex new file mode 100644 index 000000000..c982ffa3a --- /dev/null +++ b/lib/pleroma/workers/backup_worker.ex @@ -0,0 +1,17 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.BackupWorker do + alias Pleroma.Backup + + use Pleroma.Workers.WorkerHelper, queue: "backup" + + @impl Oban.Worker + def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do + with {:ok, %Backup{} = backup} <- + backup_id |> Backup.get() |> Backup.process() do + {:ok, backup} + end + end +end diff --git a/test/backup_test.exs b/test/backup_test.exs index 27f5cb7f7..5b1f76dd9 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -3,35 +3,43 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.BackupTest do + use Oban.Testing, repo: Pleroma.Repo use Pleroma.DataCase + import Pleroma.Factory import Mock alias Pleroma.Backup alias Pleroma.Bookmark alias Pleroma.Web.CommonAPI + alias Pleroma.Workers.BackupWorker - test "it creates a backup record" do + setup do: clear_config([Pleroma.Upload, :uploader]) + + test "it creates a backup record and an Oban job" do %{id: user_id} = user = insert(:user) - assert {:ok, backup} = Backup.create(user) + assert {:ok, %Oban.Job{args: args}} = Backup.create(user) + assert_enqueued(worker: BackupWorker, args: args) + backup = Backup.get(args["backup_id"]) assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup end test "it return an error if the export limit is over" do %{id: user_id} = user = insert(:user) limit_days = 7 - - assert {:ok, backup} = Backup.create(user) + assert {:ok, %Oban.Job{args: args}} = Backup.create(user) + backup = Backup.get(args["backup_id"]) assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup assert Backup.create(user) == {:error, "Last export was less than #{limit_days} days ago"} end test "it process a backup record" do + Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) %{id: user_id} = user = insert(:user) - assert {:ok, %{id: backup_id} = backup} = Backup.create(user) - assert {:ok, %Backup{} = backup} = Backup.process(backup) + assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}} = job} = Backup.create(user) + assert {:ok, backup} = BackupWorker.perform(job) assert backup.file_size > 0 assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup end From 3ad7492f9dd1c76cdbc64ad2246f8e9c8c5c4ae6 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 4 Sep 2020 18:30:39 +0400 Subject: [PATCH 14/75] Add config for Pleroma.Backup --- config/config.exs | 4 ++++ config/description.exs | 20 ++++++++++++++++++++ docs/configuration/cheatsheet.md | 5 +++++ lib/pleroma/backup.ex | 2 +- test/backup_test.exs | 2 +- 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 1f10167e5..09023e2c3 100644 --- a/config/config.exs +++ b/config/config.exs @@ -818,6 +818,10 @@ config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator +config :pleroma, Pleroma.Backup, + purge_after_days: 30, + limit_days: 7 + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/config/description.exs b/config/description.exs index 13e44afe8..4942e196d 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3712,5 +3712,25 @@ ] } ] + }, + %{ + group: :pleroma, + key: Pleroma.Backup, + type: :group, + description: "Account Backup", + children: [ + %{ + key: :purge_after_days, + type: :integer, + description: "Remove backup achives after N days", + suggestions: [30] + }, + %{ + key: :limit_days, + type: :integer, + description: "Limit user to export not more often than once per N days", + suggestions: [7] + } + ] } ] diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 42e5fe808..cc4081f14 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -1083,6 +1083,11 @@ Control favicons for instances. * `enabled`: Allow/disallow displaying and getting instances favicons +## Account Backup + +* `:purge_after_days` an integer, remove backup achives after N days. +* `:limit_days` an integer, limit user to export not more often than once per N days. + ## Frontend management Frontends in Pleroma are swappable - you can specify which one to use here. diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index 9b5d2625f..e384b6b00 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -49,7 +49,7 @@ def new(user) do defp validate_limit(user) do case get_last(user.id) do %__MODULE__{inserted_at: inserted_at} -> - days = 7 + days = Pleroma.Config.get([Pleroma.Backup, :limit_days]) diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days) if diff > days do diff --git a/test/backup_test.exs b/test/backup_test.exs index 5b1f76dd9..f343b0361 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -27,7 +27,7 @@ test "it creates a backup record and an Oban job" do test "it return an error if the export limit is over" do %{id: user_id} = user = insert(:user) - limit_days = 7 + limit_days = Pleroma.Config.get([Pleroma.Backup, :limit_days]) assert {:ok, %Oban.Job{args: args}} = Backup.create(user) backup = Backup.get(args["backup_id"]) assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup From 739cb1463ba07513f047b2ac8f7e22a16c89ef4e Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 4 Sep 2020 21:48:52 +0400 Subject: [PATCH 15/75] Add backups deletion --- lib/pleroma/backup.ex | 14 +++++++-- lib/pleroma/workers/backup_worker.ex | 37 ++++++++++++++++++++-- test/backup_test.exs | 47 +++++++++++++++++++++++++--- test/support/oban_helpers.ex | 3 ++ 4 files changed, 91 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index e384b6b00..bd50fd910 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Backup do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.UserView + alias Pleroma.Workers.BackupWorker schema "backups" do field(:content_type, :string) @@ -30,7 +31,7 @@ defmodule Pleroma.Backup do def create(user) do with :ok <- validate_limit(user), {:ok, backup} <- user |> new() |> Repo.insert() do - Pleroma.Workers.BackupWorker.enqueue("process", %{"backup_id" => backup.id}) + BackupWorker.process(backup) end end @@ -46,6 +47,14 @@ def new(user) do } end + def delete(backup) do + uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) + + with :ok <- uploader.delete_file("backups/" <> backup.file_name) do + Repo.delete(backup) + end + end + defp validate_limit(user) do case get_last(user.id) do %__MODULE__{inserted_at: inserted_at} -> @@ -75,7 +84,8 @@ def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do __MODULE__ |> where(user_id: ^user_id) |> where([b], b.id != ^latest_id) - |> Repo.delete_all() + |> Repo.all() + |> Enum.each(&BackupWorker.delete/1) end def get(id), do: Repo.get(__MODULE__, id) diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index c982ffa3a..f40020794 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -3,15 +3,46 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.BackupWorker do + use Oban.Worker, queue: :backup, max_attempts: 1 + + alias Oban.Job alias Pleroma.Backup - use Pleroma.Workers.WorkerHelper, queue: "backup" + def process(backup) do + %{"op" => "process", "backup_id" => backup.id} + |> new() + |> Oban.insert() + end + + def schedule_deletion(backup) do + days = Pleroma.Config.get([Pleroma.Backup, :purge_after_days]) + time = 60 * 60 * 24 * days + scheduled_at = Calendar.NaiveDateTime.add!(backup.inserted_at, time) + + %{"op" => "delete", "backup_id" => backup.id} + |> new(scheduled_at: scheduled_at) + |> Oban.insert() + end + + def delete(backup) do + %{"op" => "delete", "backup_id" => backup.id} + |> new() + |> Oban.insert() + end - @impl Oban.Worker def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do with {:ok, %Backup{} = backup} <- - backup_id |> Backup.get() |> Backup.process() do + backup_id |> Backup.get() |> Backup.process(), + {:ok, _job} <- schedule_deletion(backup), + :ok <- Backup.remove_outdated(backup) do {:ok, backup} end end + + def perform(%Job{args: %{"op" => "delete", "backup_id" => backup_id}}) do + case Backup.get(backup_id) do + %Backup{} = backup -> Backup.delete(backup) + nil -> :ok + end + end end diff --git a/test/backup_test.exs b/test/backup_test.exs index f343b0361..59aebe360 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -13,8 +13,12 @@ defmodule Pleroma.BackupTest do alias Pleroma.Bookmark alias Pleroma.Web.CommonAPI alias Pleroma.Workers.BackupWorker + alias Pleroma.Tests.ObanHelpers - setup do: clear_config([Pleroma.Upload, :uploader]) + setup do + clear_config([Pleroma.Upload, :uploader]) + clear_config([Pleroma.Backup, :limit_days]) + end test "it creates a backup record and an Oban job" do %{id: user_id} = user = insert(:user) @@ -38,10 +42,34 @@ test "it return an error if the export limit is over" do test "it process a backup record" do Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) %{id: user_id} = user = insert(:user) - assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}} = job} = Backup.create(user) - assert {:ok, backup} = BackupWorker.perform(job) + + assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id} = args}} = Backup.create(user) + assert {:ok, backup} = perform_job(BackupWorker, args) assert backup.file_size > 0 assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup + + delete_job_args = %{"op" => "delete", "backup_id" => backup_id} + + assert_enqueued(worker: BackupWorker, args: delete_job_args) + assert {:ok, backup} = perform_job(BackupWorker, delete_job_args) + refute Backup.get(backup_id) + end + + test "it removes outdated backups after creating a fresh one" do + Pleroma.Config.put([Pleroma.Backup, :limit_days], -1) + Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) + user = insert(:user) + + assert {:ok, job1} = Backup.create(user) + + assert {:ok, %Backup{id: backup1_id}} = ObanHelpers.perform(job1) + assert {:ok, job2} = Backup.create(user) + assert Pleroma.Repo.aggregate(Backup, :count) == 2 + assert {:ok, backup2} = ObanHelpers.perform(job2) + + ObanHelpers.perform_all() + + assert [^backup2] = Pleroma.Repo.all(Backup) end test "it creates a zip archive with user data" do @@ -145,7 +173,7 @@ test "it creates a zip archive with user data" do File.rm!(path) end - describe "it uploads a backup archive" do + describe "it uploads and deletes a backup archive" do setup do clear_config(Pleroma.Uploaders.S3, bucket: "test_bucket", @@ -173,8 +201,16 @@ test "it creates a zip archive with user data" do test "S3", %{path: path, backup: backup} do Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3) - with_mock ExAws, request: fn _ -> {:ok, :ok} end do + with_mock ExAws, + request: fn + %{http_method: :put} -> {:ok, :ok} + %{http_method: :delete} -> {:ok, %{status_code: 204}} + end do assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path) + assert {:ok, _backup} = Backup.delete(backup) + end + + with_mock ExAws, request: fn %{http_method: :delete} -> {:ok, %{status_code: 204}} end do end end @@ -182,6 +218,7 @@ test "Local", %{path: path, backup: backup} do Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path) + assert {:ok, _backup} = Backup.delete(backup) end end end diff --git a/test/support/oban_helpers.ex b/test/support/oban_helpers.ex index 9f90a821c..2468f66dc 100644 --- a/test/support/oban_helpers.ex +++ b/test/support/oban_helpers.ex @@ -7,6 +7,8 @@ defmodule Pleroma.Tests.ObanHelpers do Oban test helpers. """ + require Ecto.Query + alias Pleroma.Repo def wipe_all do @@ -15,6 +17,7 @@ def wipe_all do def perform_all do Oban.Job + |> Ecto.Query.where(state: "available") |> Repo.all() |> perform() end From abdffc6b8c2eec8f81ffe89f943f11d1f90d7074 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 4 Sep 2020 22:00:26 +0400 Subject: [PATCH 16/75] Fix Credo warning --- test/backup_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/backup_test.exs b/test/backup_test.exs index 59aebe360..5fc519eab 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -11,9 +11,9 @@ defmodule Pleroma.BackupTest do alias Pleroma.Backup alias Pleroma.Bookmark + alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.CommonAPI alias Pleroma.Workers.BackupWorker - alias Pleroma.Tests.ObanHelpers setup do clear_config([Pleroma.Upload, :uploader]) From 2c73bfe1227065fa203b0b78c9eb12cf86ab3948 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 9 Sep 2020 01:04:00 +0400 Subject: [PATCH 17/75] Add API endpoints for Backups --- lib/pleroma/backup.ex | 7 ++ .../operations/pleroma_backup_operation.ex | 79 +++++++++++++++++ .../controllers/backup_controller.ex | 27 ++++++ .../web/pleroma_api/views/backup_view.ex | 24 ++++++ lib/pleroma/web/router.ex | 3 + .../controllers/backup_controller_test.exs | 84 +++++++++++++++++++ 6 files changed, 224 insertions(+) create mode 100644 lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex create mode 100644 lib/pleroma/web/pleroma_api/controllers/backup_controller.ex create mode 100644 lib/pleroma/web/pleroma_api/views/backup_view.ex create mode 100644 test/web/pleroma_api/controllers/backup_controller_test.exs diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index bd50fd910..348e537a8 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -80,6 +80,13 @@ def get_last(user_id) do |> Repo.one() end + def list(%User{id: user_id}) do + __MODULE__ + |> where(user_id: ^user_id) + |> order_by(desc: :id) + |> Repo.all() + end + def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do __MODULE__ |> where(user_id: ^user_id) diff --git a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex new file mode 100644 index 000000000..f877ca31b --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex @@ -0,0 +1,79 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.PleromaBackupOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.ApiError + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["Backups"], + summary: "List backups", + security: [%{"oAuth" => ["read:account"]}], + operationId: "PleromaAPI.BackupController.index", + responses: %{ + 200 => + Operation.response( + "An array of backups", + "application/json", + %Schema{ + type: :array, + items: backup() + } + ), + 400 => Operation.response("Bad Request", "application/json", ApiError) + } + } + end + + def create_operation do + %Operation{ + tags: ["Backups"], + summary: "Create a backup", + security: [%{"oAuth" => ["read:account"]}], + operationId: "PleromaAPI.BackupController.create", + responses: %{ + 200 => + Operation.response( + "An array of backups", + "application/json", + %Schema{ + type: :array, + items: backup() + } + ), + 400 => Operation.response("Bad Request", "application/json", ApiError) + } + } + end + + defp backup do + %Schema{ + title: "Backup", + description: "Response schema for a backup", + type: :object, + properties: %{ + inserted_at: %Schema{type: :string, format: :"date-time"}, + content_type: %Schema{type: :string}, + file_name: %Schema{type: :string}, + file_size: %Schema{type: :integer}, + processed: %Schema{type: :boolean} + }, + example: %{ + "content_type" => "application/zip", + "file_name" => + "archive-cofe-20200908T195819-1lWrJyJqpsj8-KuHFr7N03lfsYYa5nf2NL-7A9-ddFU.zip", + "file_size" => 1024, + "inserted_at" => "2020-09-08T19:58:20", + "processed" => true + } + } + end +end diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex new file mode 100644 index 000000000..e52c77ff2 --- /dev/null +++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex @@ -0,0 +1,27 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.BackupController do + use Pleroma.Web, :controller + + alias Pleroma.Plugs.OAuthScopesPlug + + action_fallback(Pleroma.Web.MastodonAPI.FallbackController) + plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create]) + plug(OpenApiSpex.Plug.CastAndValidate, render_error: Pleroma.Web.ApiSpec.RenderError) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaBackupOperation + + def index(%{assigns: %{user: user}} = conn, _params) do + backups = Pleroma.Backup.list(user) + render(conn, "index.json", backups: backups) + end + + def create(%{assigns: %{user: user}} = conn, _params) do + with {:ok, _} <- Pleroma.Backup.create(user) do + backups = Pleroma.Backup.list(user) + render(conn, "index.json", backups: backups) + end + end +end diff --git a/lib/pleroma/web/pleroma_api/views/backup_view.ex b/lib/pleroma/web/pleroma_api/views/backup_view.ex new file mode 100644 index 000000000..02b94ce4f --- /dev/null +++ b/lib/pleroma/web/pleroma_api/views/backup_view.ex @@ -0,0 +1,24 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.BackupView do + use Pleroma.Web, :view + + alias Pleroma.Backup + alias Pleroma.Web.CommonAPI.Utils + + def render("show.json", %{backup: %Backup{} = backup}) do + %{ + content_type: backup.content_type, + file_name: backup.file_name, + file_size: backup.file_size, + processed: backup.processed, + inserted_at: Utils.to_masto_date(backup.inserted_at) + } + end + + def render("index.json", %{backups: backups}) do + render_many(backups, __MODULE__, "show.json") + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e22b31b4c..a1a5a1cb5 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -293,6 +293,9 @@ defmodule Pleroma.Web.Router do get("/accounts/mfa/setup/:method", TwoFactorAuthenticationController, :setup) post("/accounts/mfa/confirm/:method", TwoFactorAuthenticationController, :confirm) delete("/accounts/mfa/:method", TwoFactorAuthenticationController, :disable) + + get("/backups", BackupController, :index) + post("/backups", BackupController, :create) end scope "/oauth", Pleroma.Web.OAuth do diff --git a/test/web/pleroma_api/controllers/backup_controller_test.exs b/test/web/pleroma_api/controllers/backup_controller_test.exs new file mode 100644 index 000000000..1ad1b63c4 --- /dev/null +++ b/test/web/pleroma_api/controllers/backup_controller_test.exs @@ -0,0 +1,84 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do + use Pleroma.Web.ConnCase + + alias Pleroma.Backup + + setup do + clear_config([Pleroma.Upload, :uploader]) + clear_config([Backup, :limit_days]) + oauth_access(["read:accounts"]) + end + + test "GET /api/pleroma/backups", %{user: user, conn: conn} do + assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}}} = Backup.create(user) + + backup = Backup.get(backup_id) + + response = + conn + |> get("/api/pleroma/backups") + |> json_response_and_validate_schema(:ok) + + assert [ + %{ + "content_type" => "application/zip", + "file_name" => file_name, + "file_size" => 0, + "processed" => false, + "inserted_at" => _ + } + ] = response + + assert file_name == backup.file_name + + Pleroma.Tests.ObanHelpers.perform_all() + + assert [ + %{ + "file_name" => ^file_name, + "processed" => true + } + ] = + conn + |> get("/api/pleroma/backups") + |> json_response_and_validate_schema(:ok) + end + + test "POST /api/pleroma/backups", %{user: _user, conn: conn} do + assert [ + %{ + "content_type" => "application/zip", + "file_name" => file_name, + "file_size" => 0, + "processed" => false, + "inserted_at" => _ + } + ] = + conn + |> post("/api/pleroma/backups") + |> json_response_and_validate_schema(:ok) + + Pleroma.Tests.ObanHelpers.perform_all() + + assert [ + %{ + "file_name" => ^file_name, + "processed" => true + } + ] = + conn + |> get("/api/pleroma/backups") + |> json_response_and_validate_schema(:ok) + + days = Pleroma.Config.get([Backup, :limit_days]) + + assert %{"error" => "Last export was less than #{days} days ago"} == + conn + |> post("/api/pleroma/backups") + |> json_response_and_validate_schema(400) + end +end From 86ce4afd9338d81f741fa57f962509a6f0f50aff Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 9 Sep 2020 20:02:20 +0400 Subject: [PATCH 18/75] Improve backup urls --- .../api_spec/operations/pleroma_backup_operation.ex | 6 +++--- lib/pleroma/web/pleroma_api/views/backup_view.ex | 6 +++++- .../controllers/backup_controller_test.exs | 11 ++++++----- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex index f877ca31b..6993794db 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex @@ -69,9 +69,9 @@ defp backup do example: %{ "content_type" => "application/zip", "file_name" => - "archive-cofe-20200908T195819-1lWrJyJqpsj8-KuHFr7N03lfsYYa5nf2NL-7A9-ddFU.zip", - "file_size" => 1024, - "inserted_at" => "2020-09-08T19:58:20", + "https://cofe.fe:4000/media/backups/archive-foobar-20200908T164207-Yr7vuT5Wycv-sN3kSN2iJ0k-9pMo60j9qmvRCdDqIew.zip", + "file_size" => 4105, + "inserted_at" => "2020-09-08T16:42:07.000Z", "processed" => true } } diff --git a/lib/pleroma/web/pleroma_api/views/backup_view.ex b/lib/pleroma/web/pleroma_api/views/backup_view.ex index 02b94ce4f..bf40a001e 100644 --- a/lib/pleroma/web/pleroma_api/views/backup_view.ex +++ b/lib/pleroma/web/pleroma_api/views/backup_view.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupView do def render("show.json", %{backup: %Backup{} = backup}) do %{ content_type: backup.content_type, - file_name: backup.file_name, + url: download_url(backup), file_size: backup.file_size, processed: backup.processed, inserted_at: Utils.to_masto_date(backup.inserted_at) @@ -21,4 +21,8 @@ def render("show.json", %{backup: %Backup{} = backup}) do def render("index.json", %{backups: backups}) do render_many(backups, __MODULE__, "show.json") end + + def download_url(%Backup{file_name: file_name}) do + Pleroma.Web.Endpoint.url() <> "/media/backups/" <> file_name + end end diff --git a/test/web/pleroma_api/controllers/backup_controller_test.exs b/test/web/pleroma_api/controllers/backup_controller_test.exs index 1ad1b63c4..5d2f1206e 100644 --- a/test/web/pleroma_api/controllers/backup_controller_test.exs +++ b/test/web/pleroma_api/controllers/backup_controller_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do use Pleroma.Web.ConnCase alias Pleroma.Backup + alias Pleroma.Web.PleromaAPI.BackupView setup do clear_config([Pleroma.Upload, :uploader]) @@ -26,20 +27,20 @@ test "GET /api/pleroma/backups", %{user: user, conn: conn} do assert [ %{ "content_type" => "application/zip", - "file_name" => file_name, + "url" => url, "file_size" => 0, "processed" => false, "inserted_at" => _ } ] = response - assert file_name == backup.file_name + assert url == BackupView.download_url(backup) Pleroma.Tests.ObanHelpers.perform_all() assert [ %{ - "file_name" => ^file_name, + "url" => ^url, "processed" => true } ] = @@ -52,7 +53,7 @@ test "POST /api/pleroma/backups", %{user: _user, conn: conn} do assert [ %{ "content_type" => "application/zip", - "file_name" => file_name, + "url" => url, "file_size" => 0, "processed" => false, "inserted_at" => _ @@ -66,7 +67,7 @@ test "POST /api/pleroma/backups", %{user: _user, conn: conn} do assert [ %{ - "file_name" => ^file_name, + "url" => ^url, "processed" => true } ] = From cd13613db3f675b6a9171dea56fc5b03e43ae6b0 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 10 Sep 2020 20:53:06 +0400 Subject: [PATCH 19/75] Fix query --- lib/pleroma/backup.ex | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index 348e537a8..ce54a413a 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -8,6 +8,8 @@ defmodule Pleroma.Backup do import Ecto.Changeset import Ecto.Query + require Pleroma.Constants + alias Pleroma.Activity alias Pleroma.Bookmark alias Pleroma.Repo @@ -158,6 +160,7 @@ defp write_header(file, name) do "id": "#{name}.json", "type": "OrderedCollection", "orderedItems": [ + """ ) end @@ -209,13 +212,13 @@ defp statuses(dir, user) do opts = %{} |> Map.put(:type, ["Create", "Announce"]) - |> Map.put(:blocking_user, user) - |> Map.put(:muting_user, user) - |> Map.put(:reply_filtering_user, user) - |> Map.put(:announce_filtering_user, user) - |> Map.put(:user, user) + |> Map.put(:actor_id, user.ap_id) - [[user.ap_id], User.following(user), Pleroma.List.memberships(user)] + [ + [Pleroma.Constants.as_public(), user.ap_id], + User.following(user), + Pleroma.List.memberships(user) + ] |> Enum.concat() |> ActivityPub.fetch_activities_query(opts) |> write(dir, "outbox", fn a -> From 386199063b9be9fc30ad403f6afb03bf6ca47298 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 10 Sep 2020 21:09:20 +0400 Subject: [PATCH 20/75] Document `/api/pleroma/backups` API endpoint --- docs/API/pleroma_api.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md index 3fd141bd2..aeb266159 100644 --- a/docs/API/pleroma_api.md +++ b/docs/API/pleroma_api.md @@ -615,3 +615,41 @@ Emoji reactions work a lot like favourites do. They make it possible to react to {"name": "😀", "count": 2, "me": true, "accounts": [{"id" => "xyz.."...}, {"id" => "zyx..."}]} ] ``` + +## `POST /api/pleroma/backups` +### Create a user backup archive + +* Method: `POST` +* Authentication: not required +* Params: none +* Response: JSON +* Example response: + +```json +[{ + "content_type": "application/zip", + "file_size": 0, + "inserted_at": "2020-09-10T16:18:03.000Z", + "processed": false, + "url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip" +}] +``` + +## `GET /api/pleroma/backups` +### Lists user backups + +* Method: `GET` +* Authentication: not required +* Params: none +* Response: JSON +* Example response: + +```json +[{ + "content_type": "application/zip", + "file_size": 55457, + "inserted_at": "2020-09-10T16:18:03.000Z", + "processed": true, + "url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip" +}] +``` From 27bc121ec00a7b088030d6fb36c7e731f5b072b6 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 15 Sep 2020 18:07:28 +0400 Subject: [PATCH 21/75] Require email --- docs/configuration/cheatsheet.md | 3 +++ lib/pleroma/backup.ex | 19 ++++++++++++++++--- test/backup_test.exs | 16 ++++++++++++++-- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index cc4081f14..8da8a7bd6 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -1085,6 +1085,9 @@ Control favicons for instances. ## Account Backup +!!! note + Requires enabled email + * `:purge_after_days` an integer, remove backup achives after N days. * `:limit_days` an integer, limit user to export not more often than once per N days. diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index ce54a413a..3b85dd1c1 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -31,7 +31,9 @@ defmodule Pleroma.Backup do end def create(user) do - with :ok <- validate_limit(user), + with :ok <- validate_email_enabled(), + :ok <- validate_user_email(user), + :ok <- validate_limit(user), {:ok, backup} <- user |> new() |> Repo.insert() do BackupWorker.process(backup) end @@ -74,6 +76,17 @@ defp validate_limit(user) do end end + defp validate_email_enabled do + if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do + :ok + else + {:error, "Backups require enabled email"} + end + end + + defp validate_user_email(%User{email: nil}), do: {:error, "Email is required"} + defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok + def get_last(user_id) do __MODULE__ |> where(user_id: ^user_id) @@ -100,7 +113,7 @@ def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do def get(id), do: Repo.get(__MODULE__, id) def process(%__MODULE__{} = backup) do - with {:ok, zip_file} <- zip(backup), + with {:ok, zip_file} <- export(backup), {:ok, %{size: size}} <- File.stat(zip_file), {:ok, _upload} <- upload(backup, zip_file) do backup @@ -110,7 +123,7 @@ def process(%__MODULE__{} = backup) do end @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json'] - def zip(%__MODULE__{} = backup) do + def export(%__MODULE__{} = backup) do backup = Repo.preload(backup, :user) name = String.trim_trailing(backup.file_name, ".zip") dir = Path.join(System.tmp_dir!(), name) diff --git a/test/backup_test.exs b/test/backup_test.exs index 5fc519eab..318c8c419 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -18,6 +18,18 @@ defmodule Pleroma.BackupTest do setup do clear_config([Pleroma.Upload, :uploader]) clear_config([Pleroma.Backup, :limit_days]) + clear_config([Pleroma.Emails.Mailer, :enabled]) + end + + test "it requries enabled email" do + Pleroma.Config.put([Pleroma.Emails.Mailer, :enabled], false) + user = insert(:user) + assert {:error, "Backups require enabled email"} == Backup.create(user) + end + + test "it requries user's email" do + user = insert(:user, %{email: nil}) + assert {:error, "Email is required"} == Backup.create(user) end test "it creates a backup record and an Oban job" do @@ -91,7 +103,7 @@ test "it creates a zip archive with user data" do Bookmark.create(user.id, status3.id) assert {:ok, backup} = user |> Backup.new() |> Repo.insert() - assert {:ok, path} = Backup.zip(backup) + assert {:ok, path} = Backup.export(backup) assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory]) assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile) @@ -193,7 +205,7 @@ test "it creates a zip archive with user data" do Bookmark.create(user.id, status3.id) assert {:ok, backup} = user |> Backup.new() |> Repo.insert() - assert {:ok, path} = Backup.zip(backup) + assert {:ok, path} = Backup.export(backup) [path: path, backup: backup] end From e52dd62e14a956a28a706124464f3ac4b985080d Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 16 Sep 2020 23:21:13 +0400 Subject: [PATCH 22/75] Add configurable temporary directory --- config/config.exs | 3 ++- docs/configuration/cheatsheet.md | 6 ++++++ lib/pleroma/backup.ex | 7 ++++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 09023e2c3..0e12d6e15 100644 --- a/config/config.exs +++ b/config/config.exs @@ -820,7 +820,8 @@ config :pleroma, Pleroma.Backup, purge_after_days: 30, - limit_days: 7 + limit_days: 7, + dir: nil # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 8da8a7bd6..9271964f1 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -1090,6 +1090,12 @@ Control favicons for instances. * `:purge_after_days` an integer, remove backup achives after N days. * `:limit_days` an integer, limit user to export not more often than once per N days. +* `:dir` a string with a path to backup temporary directory or `nil` to let Pleroma choose temporary directory in the following order: + 1. the directory named by the TMPDIR environment variable + 2. the directory named by the TEMP environment variable + 3. the directory named by the TMP environment variable + 4. C:\TMP on Windows or /tmp on Unix-like operating systems + 5. as a last resort, the current working directory ## Frontend management diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index 3b85dd1c1..450dd5b84 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -126,7 +126,7 @@ def process(%__MODULE__{} = backup) do def export(%__MODULE__{} = backup) do backup = Repo.preload(backup, :user) name = String.trim_trailing(backup.file_name, ".zip") - dir = Path.join(System.tmp_dir!(), name) + dir = dir(name) with :ok <- File.mkdir(dir), :ok <- actor(dir, backup.user), @@ -139,6 +139,11 @@ def export(%__MODULE__{} = backup) do end end + def dir(name) do + dir = Pleroma.Config.get([__MODULE__, :dir]) || System.tmp_dir!() + Path.join(dir, name) + end + def upload(%__MODULE__{} = backup, zip_path) do uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) From 7fdd81d000d857cbcd5bf442f68c91b1c5b1cebb Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 17 Sep 2020 18:42:24 +0400 Subject: [PATCH 23/75] Add "Your backup is ready" email --- lib/pleroma/emails/user_email.ex | 16 ++++++++++++++++ lib/pleroma/workers/backup_worker.ex | 6 +++++- test/backup_test.exs | 5 ++++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index 1d8c72ae9..f943dda0d 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -189,4 +189,20 @@ def unsubscribe_url(user, notifications_type) do Router.Helpers.subscription_url(Endpoint, :unsubscribe, token) end + + def backup_is_ready_email(backup) do + %{user: user} = Pleroma.Repo.preload(backup, :user) + download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup) + + html_body = """ +

You requested a full backup of your Pleroma account. It's ready for download:

+

+ """ + + new() + |> to(recipient(user)) + |> from(sender()) + |> subject("Your account archive is ready") + |> html_body(html_body) + end end diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index f40020794..405d55269 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -34,7 +34,11 @@ def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do with {:ok, %Backup{} = backup} <- backup_id |> Backup.get() |> Backup.process(), {:ok, _job} <- schedule_deletion(backup), - :ok <- Backup.remove_outdated(backup) do + :ok <- Backup.remove_outdated(backup), + {:ok, _} <- + backup + |> Pleroma.Emails.UserEmail.backup_is_ready_email() + |> Pleroma.Emails.Mailer.deliver() do {:ok, backup} end end diff --git a/test/backup_test.exs b/test/backup_test.exs index 318c8c419..0ea40e6fd 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -6,8 +6,9 @@ defmodule Pleroma.BackupTest do use Oban.Testing, repo: Pleroma.Repo use Pleroma.DataCase - import Pleroma.Factory import Mock + import Pleroma.Factory + import Swoosh.TestAssertions alias Pleroma.Backup alias Pleroma.Bookmark @@ -65,6 +66,8 @@ test "it process a backup record" do assert_enqueued(worker: BackupWorker, args: delete_job_args) assert {:ok, backup} = perform_job(BackupWorker, delete_job_args) refute Backup.get(backup_id) + + assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup)) end test "it removes outdated backups after creating a fresh one" do From 7c22c9afb410668d87dcd4a90651d62d9a1e9e4d Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 18 Sep 2020 22:18:34 +0400 Subject: [PATCH 24/75] Allow admins request user backups --- lib/pleroma/backup.ex | 4 ++-- lib/pleroma/emails/user_email.ex | 20 +++++++++++++----- .../controllers/admin_api_controller.ex | 12 ++++++++++- lib/pleroma/web/router.ex | 2 ++ lib/pleroma/workers/backup_worker.ex | 10 +++++---- .../controllers/admin_api_controller_test.exs | 21 +++++++++++++++++++ 6 files changed, 57 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index 450dd5b84..d589f12f1 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -30,12 +30,12 @@ defmodule Pleroma.Backup do timestamps() end - def create(user) do + def create(user, admin_user_id \\ nil) do with :ok <- validate_email_enabled(), :ok <- validate_user_email(user), :ok <- validate_limit(user), {:ok, backup} <- user |> new() |> Repo.insert() do - BackupWorker.process(backup) + BackupWorker.process(backup, admin_user_id) end end diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index f943dda0d..5745794ec 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -190,14 +190,24 @@ def unsubscribe_url(user, notifications_type) do Router.Helpers.subscription_url(Endpoint, :unsubscribe, token) end - def backup_is_ready_email(backup) do + def backup_is_ready_email(backup, admin_user_id \\ nil) do %{user: user} = Pleroma.Repo.preload(backup, :user) download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup) - html_body = """ -

You requested a full backup of your Pleroma account. It's ready for download:

-

- """ + html_body = + if is_nil(admin_user_id) do + """ +

You requested a full backup of your Pleroma account. It's ready for download:

+

+ """ + else + admin = Pleroma.Repo.get(User, admin_user_id) + + """ +

Admin @#{admin.nickname} requested a full backup of your Pleroma account. It's ready for download:

+

+ """ + end new() |> to(recipient(user)) diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index d5713c3dd..f7d2fe5b1 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -23,12 +23,14 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do alias Pleroma.Web.Endpoint alias Pleroma.Web.Router + require Logger + @users_page_size 50 plug( OAuthScopesPlug, %{scopes: ["read:accounts"], admin: true} - when action in [:list_users, :user_show, :right_get, :show_user_credentials] + when action in [:list_users, :user_show, :right_get, :show_user_credentials, :create_backup] ) plug( @@ -681,6 +683,14 @@ def stats(conn, params) do json(conn, %{"status_visibility" => counters}) end + def create_backup(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do + with %User{} = user <- User.get_by_nickname(nickname), + {:ok, _} <- Pleroma.Backup.create(user, admin.id) do + Logger.info("Admin @#{admin.nickname} requested account backup for @{nickname}") + json(conn, "") + end + end + defp page_params(params) do {get_page(params["page"]), get_page_size(params["page_size"])} end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index a1a5a1cb5..e539eeeeb 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -129,6 +129,8 @@ defmodule Pleroma.Web.Router do scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do pipe_through(:admin_api) + post("/backups", AdminAPIController, :create_backup) + post("/users/follow", AdminAPIController, :user_follow) post("/users/unfollow", AdminAPIController, :user_unfollow) diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index 405d55269..65754b6a2 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -8,8 +8,8 @@ defmodule Pleroma.Workers.BackupWorker do alias Oban.Job alias Pleroma.Backup - def process(backup) do - %{"op" => "process", "backup_id" => backup.id} + def process(backup, admin_user_id \\ nil) do + %{"op" => "process", "backup_id" => backup.id, "admin_user_id" => admin_user_id} |> new() |> Oban.insert() end @@ -30,14 +30,16 @@ def delete(backup) do |> Oban.insert() end - def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do + def perform(%Job{ + args: %{"op" => "process", "backup_id" => backup_id, "admin_user_id" => admin_user_id} + }) do with {:ok, %Backup{} = backup} <- backup_id |> Backup.get() |> Backup.process(), {:ok, _job} <- schedule_deletion(backup), :ok <- Backup.remove_outdated(backup), {:ok, _} <- backup - |> Pleroma.Emails.UserEmail.backup_is_ready_email() + |> Pleroma.Emails.UserEmail.backup_is_ready_email(admin_user_id) |> Pleroma.Emails.Mailer.deliver() do {:ok, backup} end diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs index cba6b43d3..4d331779e 100644 --- a/test/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/web/admin_api/controllers/admin_api_controller_test.exs @@ -2024,6 +2024,27 @@ test "by instance", %{conn: conn} do response["status_visibility"] end end + + describe "/api/pleroma/backups" do + test "it creates a backup", %{conn: conn} do + admin = insert(:user, is_admin: true) + token = insert(:oauth_admin_token, user: admin) + user = insert(:user) + + assert "" == + conn + |> assign(:user, admin) + |> assign(:token, token) + |> post("/api/pleroma/admin/backups", %{nickname: user.nickname}) + |> json_response(200) + + assert [backup] = Repo.all(Pleroma.Backup) + + ObanHelpers.perform_all() + + assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id)) + end + end end # Needed for testing From 563801716a0aa54e30f680b4e985d4b8c79578fb Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 18 Sep 2020 22:01:46 +0400 Subject: [PATCH 25/75] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fc1750d1..04b49d80a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`) - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`) - Mix task option for force-unfollowing relays +- Account backup ### Changed From e50314d9d342dbf9a03ca484654b07717592d4bd Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 18 Sep 2020 22:33:12 +0400 Subject: [PATCH 26/75] Fix export --- lib/pleroma/backup.ex | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index d589f12f1..242773bdb 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -191,16 +191,13 @@ defp write(query, dir, name, fun) do counter = :counters.new(1, []) query - |> Pleroma.RepoStreamer.chunk_stream(100) - |> Stream.each(fn items -> - Enum.each(items, fn i -> - with {:ok, str} <- fun.(i), - :ok <- IO.write(file, str <> ",\n") do - :counters.add(counter, 1, 1) - end - end) + |> Pleroma.Repo.chunk_stream(100) + |> Enum.each(fn i -> + with {:ok, str} <- fun.(i), + :ok <- IO.write(file, str <> ",\n") do + :counters.add(counter, 1, 1) + end end) - |> Stream.run() total = :counters.get(counter, 1) From a9efd441e242f1d8ac608b866d0cfafe4833243a Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sun, 20 Sep 2020 19:57:09 +0400 Subject: [PATCH 27/75] Use `Pleroma.Repo.chunk_stream/2` instead of `Pleroma.RepoStreamer.chunk_stream/2` --- lib/pleroma/backup.ex | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index 242773bdb..f5f39431d 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -188,18 +188,17 @@ defp write(query, dir, name, fun) do with {:ok, file} <- File.open(path, [:write, :utf8]), :ok <- write_header(file, name) do - counter = :counters.new(1, []) - - query - |> Pleroma.Repo.chunk_stream(100) - |> Enum.each(fn i -> - with {:ok, str} <- fun.(i), - :ok <- IO.write(file, str <> ",\n") do - :counters.add(counter, 1, 1) - end - end) - - total = :counters.get(counter, 1) + total = + query + |> Pleroma.Repo.chunk_stream(100) + |> Enum.reduce(0, fn i, acc -> + with {:ok, str} <- fun.(i), + :ok <- IO.write(file, str <> ",\n") do + acc + 1 + else + _ -> acc + end + end) with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n \"totalItems\": #{total}}") do File.close(file) From 17562bf4147ab03e171b1f1d365a512f2e5b3202 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sun, 20 Sep 2020 20:43:27 +0400 Subject: [PATCH 28/75] Move API endpoints to `/api/v1/pleroma/backups` --- docs/API/pleroma_api.md | 4 ++-- lib/pleroma/web/router.ex | 6 +++--- .../controllers/backup_controller_test.exs | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md index aeb266159..fa3a9a449 100644 --- a/docs/API/pleroma_api.md +++ b/docs/API/pleroma_api.md @@ -616,7 +616,7 @@ Emoji reactions work a lot like favourites do. They make it possible to react to ] ``` -## `POST /api/pleroma/backups` +## `POST /api/v1/pleroma/backups` ### Create a user backup archive * Method: `POST` @@ -635,7 +635,7 @@ Emoji reactions work a lot like favourites do. They make it possible to react to }] ``` -## `GET /api/pleroma/backups` +## `GET /api/v1/pleroma/backups` ### Lists user backups * Method: `GET` diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e539eeeeb..ad7e315c7 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -295,9 +295,6 @@ defmodule Pleroma.Web.Router do get("/accounts/mfa/setup/:method", TwoFactorAuthenticationController, :setup) post("/accounts/mfa/confirm/:method", TwoFactorAuthenticationController, :confirm) delete("/accounts/mfa/:method", TwoFactorAuthenticationController, :disable) - - get("/backups", BackupController, :index) - post("/backups", BackupController, :create) end scope "/oauth", Pleroma.Web.OAuth do @@ -358,6 +355,9 @@ defmodule Pleroma.Web.Router do put("/mascot", MascotController, :update) post("/scrobble", ScrobbleController, :create) + + get("/backups", BackupController, :index) + post("/backups", BackupController, :create) end scope [] do diff --git a/test/web/pleroma_api/controllers/backup_controller_test.exs b/test/web/pleroma_api/controllers/backup_controller_test.exs index 5d2f1206e..b2ac74c7d 100644 --- a/test/web/pleroma_api/controllers/backup_controller_test.exs +++ b/test/web/pleroma_api/controllers/backup_controller_test.exs @@ -14,14 +14,14 @@ defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do oauth_access(["read:accounts"]) end - test "GET /api/pleroma/backups", %{user: user, conn: conn} do + test "GET /api/v1/pleroma/backups", %{user: user, conn: conn} do assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}}} = Backup.create(user) backup = Backup.get(backup_id) response = conn - |> get("/api/pleroma/backups") + |> get("/api/v1/pleroma/backups") |> json_response_and_validate_schema(:ok) assert [ @@ -45,11 +45,11 @@ test "GET /api/pleroma/backups", %{user: user, conn: conn} do } ] = conn - |> get("/api/pleroma/backups") + |> get("/api/v1/pleroma/backups") |> json_response_and_validate_schema(:ok) end - test "POST /api/pleroma/backups", %{user: _user, conn: conn} do + test "POST /api/v1/pleroma/backups", %{user: _user, conn: conn} do assert [ %{ "content_type" => "application/zip", @@ -60,7 +60,7 @@ test "POST /api/pleroma/backups", %{user: _user, conn: conn} do } ] = conn - |> post("/api/pleroma/backups") + |> post("/api/v1/pleroma/backups") |> json_response_and_validate_schema(:ok) Pleroma.Tests.ObanHelpers.perform_all() @@ -72,14 +72,14 @@ test "POST /api/pleroma/backups", %{user: _user, conn: conn} do } ] = conn - |> get("/api/pleroma/backups") + |> get("/api/v1/pleroma/backups") |> json_response_and_validate_schema(:ok) days = Pleroma.Config.get([Backup, :limit_days]) assert %{"error" => "Last export was less than #{days} days ago"} == conn - |> post("/api/pleroma/backups") + |> post("/api/v1/pleroma/backups") |> json_response_and_validate_schema(400) end end From e4792ce76af3094d378a3a201ca429ae38203696 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sun, 20 Sep 2020 21:06:16 +0400 Subject: [PATCH 29/75] Do not limit admins --- lib/pleroma/backup.ex | 10 ++++---- .../controllers/admin_api_controller_test.exs | 24 +++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index f5f39431d..e2673db80 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -30,12 +30,12 @@ defmodule Pleroma.Backup do timestamps() end - def create(user, admin_user_id \\ nil) do + def create(user, admin_id \\ nil) do with :ok <- validate_email_enabled(), :ok <- validate_user_email(user), - :ok <- validate_limit(user), + :ok <- validate_limit(user, admin_id), {:ok, backup} <- user |> new() |> Repo.insert() do - BackupWorker.process(backup, admin_user_id) + BackupWorker.process(backup, admin_id) end end @@ -59,7 +59,9 @@ def delete(backup) do end end - defp validate_limit(user) do + defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok + + defp validate_limit(user, nil) do case get_last(user.id) do %__MODULE__{inserted_at: inserted_at} -> days = Pleroma.Config.get([Pleroma.Backup, :limit_days]) diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs index 4d331779e..4b3abce0d 100644 --- a/test/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/web/admin_api/controllers/admin_api_controller_test.exs @@ -2044,6 +2044,30 @@ test "it creates a backup", %{conn: conn} do assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id)) end + + test "it doesn't limit admins", %{conn: conn} do + admin = insert(:user, is_admin: true) + token = insert(:oauth_admin_token, user: admin) + user = insert(:user) + + assert "" == + conn + |> assign(:user, admin) + |> assign(:token, token) + |> post("/api/pleroma/admin/backups", %{nickname: user.nickname}) + |> json_response(200) + + assert [_backup] = Repo.all(Pleroma.Backup) + + assert "" == + conn + |> assign(:user, admin) + |> assign(:token, token) + |> post("/api/pleroma/admin/backups", %{nickname: user.nickname}) + |> json_response(200) + + assert Repo.aggregate(Pleroma.Backup, :count) == 2 + end end end From 8baee855d90530def46dc62b81e6a0cb0c315914 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 21 Sep 2020 21:47:36 +0400 Subject: [PATCH 30/75] Fix emails --- lib/pleroma/emails/user_email.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index 5745794ec..806a61fd2 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -198,14 +198,14 @@ def backup_is_ready_email(backup, admin_user_id \\ nil) do if is_nil(admin_user_id) do """

You requested a full backup of your Pleroma account. It's ready for download:

-

+

#{download_url}

""" else admin = Pleroma.Repo.get(User, admin_user_id) """

Admin @#{admin.nickname} requested a full backup of your Pleroma account. It's ready for download:

-

+

#{download_url}

""" end From f1e4333dd7976e8cbef44a3bcfe5c96bef177c6f Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 23 Sep 2020 20:23:11 +0400 Subject: [PATCH 31/75] Fix test --- test/backup_test.exs | 7 ++++++- .../admin_api/controllers/admin_api_controller_test.exs | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/test/backup_test.exs b/test/backup_test.exs index 0ea40e6fd..23c08b680 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -67,7 +67,12 @@ test "it process a backup record" do assert {:ok, backup} = perform_job(BackupWorker, delete_job_args) refute Backup.get(backup_id) - assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup)) + email = Pleroma.Emails.UserEmail.backup_is_ready_email(backup) + + assert_email_sent( + to: {user.name, user.email}, + html_body: email.html_body + ) end test "it removes outdated backups after creating a fresh one" do diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs index 4b3abce0d..a6dc4f62d 100644 --- a/test/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/web/admin_api/controllers/admin_api_controller_test.exs @@ -2042,7 +2042,10 @@ test "it creates a backup", %{conn: conn} do ObanHelpers.perform_all() - assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id)) + email = Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id) + + assert String.contains?(email.html_body, "Admin @#{admin.nickname} requested a full backup") + assert_email_sent(to: {user.name, user.email}, html_body: email.html_body) end test "it doesn't limit admins", %{conn: conn} do From 6d5f02a1da81ed7693c5ae364a25bc0b54ee1a38 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 26 Sep 2020 20:34:44 +0400 Subject: [PATCH 32/75] Fix API documentation --- docs/API/pleroma_api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md index fa3a9a449..7a0a80dad 100644 --- a/docs/API/pleroma_api.md +++ b/docs/API/pleroma_api.md @@ -620,7 +620,7 @@ Emoji reactions work a lot like favourites do. They make it possible to react to ### Create a user backup archive * Method: `POST` -* Authentication: not required +* Authentication: required * Params: none * Response: JSON * Example response: From d7a5291b4fa3b7568674c0f7643fe287fcd21eff Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 26 Sep 2020 21:24:35 +0400 Subject: [PATCH 33/75] Use `Jason.encode/1` for likes and bookmarks --- lib/pleroma/backup.ex | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index e2673db80..b43dc94d6 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -194,7 +194,8 @@ defp write(query, dir, name, fun) do query |> Pleroma.Repo.chunk_stream(100) |> Enum.reduce(0, fn i, acc -> - with {:ok, str} <- fun.(i), + with {:ok, data} <- fun.(i), + {:ok, str} <- Jason.encode(data), :ok <- IO.write(file, str <> ",\n") do acc + 1 else @@ -213,7 +214,7 @@ defp bookmarks(dir, %{id: user_id} = _user) do |> where(user_id: ^user_id) |> join(:inner, [b], activity in assoc(b, :activity)) |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)}) - |> write(dir, "bookmarks", fn a -> {:ok, "\"#{a.object}\""} end) + |> write(dir, "bookmarks", fn a -> {:ok, a.object} end) end defp likes(dir, user) do @@ -221,7 +222,7 @@ defp likes(dir, user) do |> Activity.Queries.by_actor() |> Activity.Queries.by_type("Like") |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)}) - |> write(dir, "likes", fn a -> {:ok, "\"#{a.object}\""} end) + |> write(dir, "likes", fn a -> {:ok, a.object} end) end defp statuses(dir, user) do @@ -239,7 +240,7 @@ defp statuses(dir, user) do |> ActivityPub.fetch_activities_query(opts) |> write(dir, "outbox", fn a -> with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do - activity |> Map.delete("@context") |> Jason.encode() + {:ok, Map.delete(activity, "@context")} end end) end From 9af9f02f4b3c4eac859a69ab9b2f546a91110287 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 26 Sep 2020 21:45:03 +0400 Subject: [PATCH 34/75] Use Gettext for error messages --- lib/pleroma/backup.ex | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index b43dc94d6..0ebaf02e5 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Backup do import Ecto.Changeset import Ecto.Query + import Pleroma.Web.Gettext require Pleroma.Constants @@ -70,7 +71,14 @@ defp validate_limit(user, nil) do if diff > days do :ok else - {:error, "Last export was less than #{days} days ago"} + {:error, + dngettext( + "errors", + "Last export was less than a day ago", + "Last export was less than %{days} days ago", + days, + days: days + )} end nil -> @@ -82,11 +90,14 @@ defp validate_email_enabled do if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do :ok else - {:error, "Backups require enabled email"} + {:error, dgettext("errors", "Backups require enabled email")} end end - defp validate_user_email(%User{email: nil}), do: {:error, "Email is required"} + defp validate_user_email(%User{email: nil}) do + {:error, dgettext("errors", "Email is required")} + end + defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok def get_last(user_id) do From 08972dd135c200073f5de0c8731b886cc2e72eeb Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 26 Sep 2020 21:50:31 +0400 Subject: [PATCH 35/75] Use Path.join/2 --- lib/pleroma/backup.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index 0ebaf02e5..cee51d7c1 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -55,7 +55,7 @@ def new(user) do def delete(backup) do uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) - with :ok <- uploader.delete_file("backups/" <> backup.file_name) do + with :ok <- uploader.delete_file(Path.join("backups", backup.file_name)) do Repo.delete(backup) end end @@ -164,7 +164,7 @@ def upload(%__MODULE__{} = backup, zip_path) do name: backup.file_name, tempfile: zip_path, content_type: backup.content_type, - path: "backups/" <> backup.file_name + path: Path.join("backups", backup.file_name) } with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload), @@ -178,7 +178,7 @@ defp actor(dir, user) do UserView.render("user.json", %{user: user}) |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"}) |> Jason.encode() do - File.write(dir <> "/actor.json", json) + File.write(Path.join(dir, "actor.json"), json) end end @@ -197,7 +197,7 @@ defp write_header(file, name) do end defp write(query, dir, name, fun) do - path = dir <> "/#{name}.json" + path = Path.join(dir, "#{name}.json") with {:ok, file} <- File.open(path, [:write, :utf8]), :ok <- write_header(file, name) do From 8545d533ddee2978e9bf7f3284cc7dcb822a77e6 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 26 Sep 2020 21:53:04 +0400 Subject: [PATCH 36/75] Use to_string/1 instead of :binary.list_to_bin/1 --- lib/pleroma/backup.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index cee51d7c1..629e879a7 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -148,7 +148,7 @@ def export(%__MODULE__{} = backup) do :ok <- bookmarks(dir, backup.user), {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir), {:ok, _} <- File.rm_rf(dir) do - {:ok, :binary.list_to_bin(zip_path)} + {:ok, to_string(zip_path)} end end From bc3db724030707e9903d161a70b10fe217a83212 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 26 Sep 2020 23:16:56 +0400 Subject: [PATCH 37/75] Use ModerationLog instead of Logger --- lib/pleroma/moderation_log.ex | 10 ++++++++ .../controllers/admin_api_controller.ex | 3 ++- .../controllers/admin_api_controller_test.exs | 23 +++++++++++++++++-- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex index 47036a6f6..be1e81467 100644 --- a/lib/pleroma/moderation_log.ex +++ b/lib/pleroma/moderation_log.ex @@ -651,6 +651,16 @@ def get_log_entry_message(%ModerationLog{ "@#{actor_nickname} deleted chat message ##{subject_id}" end + def get_log_entry_message(%ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor_nickname}, + "action" => "create_backup", + "subject" => %{"nickname" => user_nickname} + } + }) do + "@#{actor_nickname} requested account backup for @#{user_nickname}" + end + defp nicknames_to_string(nicknames) do nicknames |> Enum.map(&"@#{&1}") diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index f7d2fe5b1..8b5310d80 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -686,7 +686,8 @@ def stats(conn, params) do def create_backup(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do with %User{} = user <- User.get_by_nickname(nickname), {:ok, _} <- Pleroma.Backup.create(user, admin.id) do - Logger.info("Admin @#{admin.nickname} requested account backup for @{nickname}") + ModerationLog.insert_log(%{actor: admin, subject: user, action: "create_backup"}) + json(conn, "") end end diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs index a6dc4f62d..34d48c2c1 100644 --- a/test/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/web/admin_api/controllers/admin_api_controller_test.exs @@ -2027,9 +2027,9 @@ test "by instance", %{conn: conn} do describe "/api/pleroma/backups" do test "it creates a backup", %{conn: conn} do - admin = insert(:user, is_admin: true) + admin = %{id: admin_id, nickname: admin_nickname} = insert(:user, is_admin: true) token = insert(:oauth_admin_token, user: admin) - user = insert(:user) + user = %{id: user_id, nickname: user_nickname} = insert(:user) assert "" == conn @@ -2046,6 +2046,25 @@ test "it creates a backup", %{conn: conn} do assert String.contains?(email.html_body, "Admin @#{admin.nickname} requested a full backup") assert_email_sent(to: {user.name, user.email}, html_body: email.html_body) + + log_message = "@#{admin_nickname} requested account backup for @#{user_nickname}" + + assert [ + %{ + data: %{ + "action" => "create_backup", + "actor" => %{ + "id" => ^admin_id, + "nickname" => ^admin_nickname + }, + "message" => ^log_message, + "subject" => %{ + "id" => ^user_id, + "nickname" => ^user_nickname + } + } + } + ] = Pleroma.ModerationLog |> Repo.all() end test "it doesn't limit admins", %{conn: conn} do From e1eb54d3899883b5af6a43687a2345543d69ad4a Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 11 Oct 2020 13:37:19 +0300 Subject: [PATCH 38/75] [#3053] Rollback of access control changes in ActivityPubController (base actions: :user, :object, :activity). --- .../activity_pub/activity_pub_controller.ex | 56 +++++++++---------- 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index c78edfb4c..732c44271 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -32,23 +32,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do @federating_only_actions [:internal_fetch, :relay, :relay_following, :relay_followers] - # Note: :following and :followers must be served even without authentication (as via :api) - @auth_only_actions [:read_inbox, :update_outbox, :whoami, :upload_media] - - # Always accessible actions (must perform entity accessibility checks) - @no_auth_no_federation_actions [:user, :activity, :object] - - @authenticated_or_federating_actions @federating_only_actions ++ - @auth_only_actions ++ @no_auth_no_federation_actions - plug(FederatingPlug when action in @federating_only_actions) - plug(EnsureAuthenticatedPlug when action in @auth_only_actions) - plug( EnsureAuthenticatedPlug, - [unless_func: &FederatingPlug.federating?/1] - when action not in @authenticated_or_federating_actions + [unless_func: &FederatingPlug.federating?/1] when action not in @federating_only_actions + ) + + # Note: :following and :followers must be served even without authentication (as via :api) + plug( + EnsureAuthenticatedPlug + when action in [:read_inbox, :update_outbox, :whoami, :upload_media] ) plug( @@ -72,22 +66,21 @@ defp relay_active?(conn, _) do def user(conn, %{"nickname" => nickname}) do with %User{local: true} = user <- User.get_cached_by_nickname(nickname), - {_, :visible} <- {:visibility, User.visible_for(user, _reading_user = nil)}, {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_content_type("application/activity+json") |> put_view(UserView) |> render("user.json", %{user: user}) else - _ -> {:error, :not_found} + nil -> {:error, :not_found} + %{local: false} -> {:error, :not_found} end end def object(conn, _) do with ap_id <- Endpoint.url() <> conn.request_path, %Object{} = object <- Object.get_cached_by_ap_id(ap_id), - {_, true} <- {:public?, Visibility.is_public?(object)}, - {_, false} <- {:restricted?, Visibility.restrict_unauthenticated_access?(object)} do + {_, true} <- {:public?, Visibility.is_public?(object)} do conn |> assign(:tracking_fun_data, object.id) |> set_cache_ttl_for(object) @@ -95,15 +88,25 @@ def object(conn, _) do |> put_view(ObjectView) |> render("object.json", object: object) else - _ -> {:error, :not_found} + {:public?, false} -> + {:error, :not_found} end end + def track_object_fetch(conn, nil), do: conn + + def track_object_fetch(conn, object_id) do + with %{assigns: %{user: %User{id: user_id}}} <- conn do + Delivery.create(object_id, user_id) + end + + conn + end + def activity(conn, _params) do with ap_id <- Endpoint.url() <> conn.request_path, %Activity{} = activity <- Activity.normalize(ap_id), - {_, true} <- {:public?, Visibility.is_public?(activity)}, - {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)} do + {_, true} <- {:public?, Visibility.is_public?(activity)} do conn |> maybe_set_tracking_data(activity) |> set_cache_ttl_for(activity) @@ -111,7 +114,8 @@ def activity(conn, _params) do |> put_view(ObjectView) |> render("object.json", object: activity) else - _ -> {:error, :not_found} + {:public?, false} -> {:error, :not_found} + nil -> {:error, :not_found} end end @@ -546,14 +550,4 @@ def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = |> json(object.data) end end - - def track_object_fetch(conn, nil), do: conn - - def track_object_fetch(conn, object_id) do - with %{assigns: %{user: %User{id: user_id}}} <- conn do - Delivery.create(object_id, user_id) - end - - conn - end end From 89c595b772eaaa8809f5339d708d7dc22e51b662 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 11 Oct 2020 22:34:28 +0300 Subject: [PATCH 39/75] [#3053] Removed target accessibility checks for OStatus endpoints delegating to RedirectController. Added tests. --- CHANGELOG.md | 1 + lib/pleroma/web/ostatus/ostatus_controller.ex | 13 +++---- lib/pleroma/web/router.ex | 38 +++++++++---------- .../static_fe/static_fe_controller_test.exs | 23 +++++++++++ 4 files changed, 48 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ae5d0eda..1e7bcca08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ switched to a new configuration mechanism, however it was not officially removed - Add documented-but-missing chat pagination. - Allow sending out emails again. +- OStatus / static FE endpoints: fixed inaccessibility for anonymous users on non-federating instances, switched to handling per `:restrict_unauthenticated` setting. ## Unreleased (Patch) diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index b4dc2a87f..e03ca8c0a 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -37,11 +37,10 @@ def object(conn, _params) do with id <- Endpoint.url() <> conn.request_path, {_, %Activity{} = activity} <- {:activity, Activity.get_create_by_object_ap_id_with_object(id)}, - {_, true} <- {:public?, Visibility.is_public?(activity)}, - {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)} do + {_, true} <- {:public?, Visibility.is_public?(activity)} do redirect(conn, to: "/notice/#{activity.id}") else - reason when reason in [{:public?, false}, {:visible?, false}, {:activity, nil}] -> + reason when reason in [{:public?, false}, {:activity, nil}] -> {:error, :not_found} e -> @@ -57,11 +56,10 @@ def activity(%{assigns: %{format: format}} = conn, _params) def activity(conn, _params) do with id <- Endpoint.url() <> conn.request_path, {_, %Activity{} = activity} <- {:activity, Activity.normalize(id)}, - {_, true} <- {:public?, Visibility.is_public?(activity)}, - {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)} do + {_, true} <- {:public?, Visibility.is_public?(activity)} do redirect(conn, to: "/notice/#{activity.id}") else - reason when reason in [{:public?, false}, {:visible?, false}, {:activity, nil}] -> + reason when reason in [{:public?, false}, {:activity, nil}] -> {:error, :not_found} e -> @@ -72,7 +70,6 @@ def activity(conn, _params) do def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id_with_object(id)}, {_, true} <- {:public?, Visibility.is_public?(activity)}, - {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)}, %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do cond do format in ["json", "activity+json"] -> @@ -100,7 +97,7 @@ def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do RedirectController.redirector(conn, nil) end else - reason when reason in [{:public?, false}, {:visible?, false}, {:activity, nil}] -> + reason when reason in [{:public?, false}, {:activity, nil}] -> conn |> put_status(404) |> RedirectController.redirector(nil, 404) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 97fcaafd5..ef56360ed 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -9,6 +9,18 @@ defmodule Pleroma.Web.Router do plug(:accepts, ["html"]) end + pipeline :accepts_html_xml do + plug(:accepts, ["html", "xml", "rss", "atom"]) + end + + pipeline :accepts_html_json do + plug(:accepts, ["html", "activity+json", "json"]) + end + + pipeline :accepts_html_xml_json do + plug(:accepts, ["html", "xml", "rss", "atom", "activity+json", "json"]) + end + pipeline :accepts_xml_rss_atom do plug(:accepts, ["xml", "rss", "atom"]) end @@ -574,24 +586,10 @@ defmodule Pleroma.Web.Router do ) end - pipeline :ostatus_html_json do - plug(:accepts, ["html", "activity+json", "json"]) - plug(Pleroma.Plugs.StaticFEPlug) - end - - pipeline :ostatus_html_xml do - plug(:accepts, ["html", "xml", "rss", "atom"]) - plug(Pleroma.Plugs.StaticFEPlug) - end - - pipeline :ostatus_html_xml_json do - plug(:accepts, ["html", "xml", "rss", "atom", "activity+json", "json"]) - plug(Pleroma.Plugs.StaticFEPlug) - end - scope "/", Pleroma.Web do # Note: html format is supported only if static FE is enabled - pipe_through(:ostatus_html_json) + # Note: http signature is only considered for json requests (no auth for non-json requests) + pipe_through([:accepts_html_json, :http_signature, Pleroma.Plugs.StaticFEPlug]) get("/objects/:uuid", OStatus.OStatusController, :object) get("/activities/:uuid", OStatus.OStatusController, :activity) @@ -604,15 +602,17 @@ defmodule Pleroma.Web.Router do scope "/", Pleroma.Web do # Note: html format is supported only if static FE is enabled - pipe_through(:ostatus_html_xml_json) + # Note: http signature is only considered for json requests (no auth for non-json requests) + pipe_through([:accepts_html_xml_json, :http_signature, Pleroma.Plugs.StaticFEPlug]) - # Note: for json format responds with user profile (not user feed) + # Note: returns user _profile_ for json requests, redirects to user _feed_ for non-json ones get("/users/:nickname", Feed.UserController, :feed_redirect, as: :user_feed) end scope "/", Pleroma.Web do # Note: html format is supported only if static FE is enabled - pipe_through(:ostatus_html_xml) + pipe_through([:accepts_html_xml, Pleroma.Plugs.StaticFEPlug]) + get("/users/:nickname/feed", Feed.UserController, :feed, as: :user_feed) end diff --git a/test/web/static_fe/static_fe_controller_test.exs b/test/web/static_fe/static_fe_controller_test.exs index bab0b0a7b..8baf5b1ce 100644 --- a/test/web/static_fe/static_fe_controller_test.exs +++ b/test/web/static_fe/static_fe_controller_test.exs @@ -78,6 +78,18 @@ test "does not require authentication on non-federating instances", %{ assert html_response(conn, 200) =~ user.nickname end + + test "returns 404 for local user with `restrict_unauthenticated/profiles/local` setting", %{ + conn: conn + } do + clear_config([:restrict_unauthenticated, :profiles, :local], true) + + local_user = insert(:user, local: true) + + conn + |> get("/users/#{local_user.nickname}") + |> html_response(404) + end end describe "notice html" do @@ -200,5 +212,16 @@ test "does not require authentication on non-federating instances", %{ assert html_response(conn, 200) =~ "testing a thing!" end + + test "returns 404 for local public activity with `restrict_unauthenticated/activities/local` setting", + %{conn: conn, user: user} do + clear_config([:restrict_unauthenticated, :activities, :local], true) + + {:ok, activity} = CommonAPI.post(user, %{status: "testing a thing!"}) + + conn + |> get("/notice/#{activity.id}") + |> html_response(404) + end end end From 33f4f39b1cf3a6d8ce350da194696b19ca6f3a05 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 13 Oct 2020 21:39:41 +0400 Subject: [PATCH 40/75] Add pagination for Blocks --- .../api_spec/operations/account_operation.ex | 1 + .../controllers/account_controller.ex | 12 +++-- .../controllers/account_controller_test.exs | 46 +++++++++++++++++-- 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index d90ddb787..9cd516f05 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -348,6 +348,7 @@ def blocks_operation do operationId: "AccountController.blocks", description: "View your blocks. See also accounts/:id/{block,unblock}", security: [%{"oAuth" => ["read:blocks"]}], + parameters: pagination_params(), responses: %{ 200 => Operation.response("Accounts", "application/json", array_of_accounts()) } diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 4f9696d52..1b221e3a1 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -448,9 +448,15 @@ def mutes(%{assigns: %{user: user}} = conn, _) do end @doc "GET /api/v1/blocks" - def blocks(%{assigns: %{user: user}} = conn, _) do - users = User.blocked_users(user, _restrict_deactivated = true) - render(conn, "index.json", users: users, for: user, as: :user) + def blocks(%{assigns: %{user: user}} = conn, params) do + users = + user + |> User.blocked_users_relation(_restrict_deactivated = true) + |> Pleroma.Pagination.fetch_paginated(Map.put(params, :skip_order, true)) + + conn + |> add_link_headers(users) + |> render("index.json", users: users, for: user, as: :user) end @doc "GET /api/v1/endorsements" diff --git a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs index f7f1369e4..6ad9dfc39 100644 --- a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs @@ -1521,16 +1521,52 @@ test "getting a list of mutes" do test "getting a list of blocks" do %{user: user, conn: conn} = oauth_access(["read:blocks"]) - other_user = insert(:user) + %{id: id1} = other_user1 = insert(:user) + %{id: id2} = other_user2 = insert(:user) + %{id: id3} = other_user3 = insert(:user) - {:ok, _user_relationship} = User.block(user, other_user) + {:ok, _user_relationship} = User.block(user, other_user1) + {:ok, _user_relationship} = User.block(user, other_user3) + {:ok, _user_relationship} = User.block(user, other_user2) - conn = + result = conn |> assign(:user, user) |> get("/api/v1/blocks") + |> json_response_and_validate_schema(200) - other_user_id = to_string(other_user.id) - assert [%{"id" => ^other_user_id}] = json_response_and_validate_schema(conn, 200) + assert [id1, id2, id3] == Enum.map(result, & &1["id"]) + + result = + conn + |> assign(:user, user) + |> get("/api/v1/blocks?limit=1") + |> json_response_and_validate_schema(200) + + assert [%{"id" => ^id1}] = result + + result = + conn + |> assign(:user, user) + |> get("/api/v1/blocks?since_id=#{id1}") + |> json_response_and_validate_schema(200) + + assert [%{"id" => ^id2}, %{"id" => ^id3}] = result + + result = + conn + |> assign(:user, user) + |> get("/api/v1/blocks?since_id=#{id1}&max_id=#{id3}") + |> json_response_and_validate_schema(200) + + assert [%{"id" => ^id2}] = result + + result = + conn + |> assign(:user, user) + |> get("/api/v1/blocks?since_id=#{id1}&limit=1") + |> json_response_and_validate_schema(200) + + assert [%{"id" => ^id2}] = result end end From 6734abcbd448b92d57ec376b796fea1fad18b792 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 13 Oct 2020 21:58:18 +0400 Subject: [PATCH 41/75] Add pagination for Mutes --- .../api_spec/operations/account_operation.ex | 1 + .../controllers/account_controller.ex | 12 +++-- .../controllers/account_controller_test.exs | 49 +++++++++++++++++-- 3 files changed, 54 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index 9cd516f05..4934b7788 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -335,6 +335,7 @@ def mutes_operation do operationId: "AccountController.mutes", description: "Accounts the user has muted.", security: [%{"oAuth" => ["follow", "read:mutes"]}], + parameters: pagination_params(), responses: %{ 200 => Operation.response("Accounts", "application/json", array_of_accounts()) } diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 1b221e3a1..c8606e5d6 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -442,9 +442,15 @@ def follow_by_uri(%{body_params: %{uri: uri}} = conn, _) do end @doc "GET /api/v1/mutes" - def mutes(%{assigns: %{user: user}} = conn, _) do - users = User.muted_users(user, _restrict_deactivated = true) - render(conn, "index.json", users: users, for: user, as: :user) + def mutes(%{assigns: %{user: user}} = conn, params) do + users = + user + |> User.muted_users_relation(_restrict_deactivated = true) + |> Pleroma.Pagination.fetch_paginated(Map.put(params, :skip_order, true)) + + conn + |> add_link_headers(users) + |> render("index.json", users: users, for: user, as: :user) end @doc "GET /api/v1/blocks" diff --git a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs index 6ad9dfc39..69f2b6f4a 100644 --- a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs @@ -1509,14 +1509,53 @@ test "returns an empty list on a bad request", %{conn: conn} do test "getting a list of mutes" do %{user: user, conn: conn} = oauth_access(["read:mutes"]) - other_user = insert(:user) + %{id: id1} = other_user1 = insert(:user) + %{id: id2} = other_user2 = insert(:user) + %{id: id3} = other_user3 = insert(:user) - {:ok, _user_relationships} = User.mute(user, other_user) + {:ok, _user_relationships} = User.mute(user, other_user1) + {:ok, _user_relationships} = User.mute(user, other_user2) + {:ok, _user_relationships} = User.mute(user, other_user3) - conn = get(conn, "/api/v1/mutes") + result = + conn + |> assign(:user, user) + |> get("/api/v1/mutes") + |> json_response_and_validate_schema(200) - other_user_id = to_string(other_user.id) - assert [%{"id" => ^other_user_id}] = json_response_and_validate_schema(conn, 200) + assert [id1, id2, id3] == Enum.map(result, & &1["id"]) + + result = + conn + |> assign(:user, user) + |> get("/api/v1/mutes?limit=1") + |> json_response_and_validate_schema(200) + + assert [%{"id" => ^id1}] = result + + result = + conn + |> assign(:user, user) + |> get("/api/v1/mutes?since_id=#{id1}") + |> json_response_and_validate_schema(200) + + assert [%{"id" => ^id2}, %{"id" => ^id3}] = result + + result = + conn + |> assign(:user, user) + |> get("/api/v1/mutes?since_id=#{id1}&max_id=#{id3}") + |> json_response_and_validate_schema(200) + + assert [%{"id" => ^id2}] = result + + result = + conn + |> assign(:user, user) + |> get("/api/v1/mutes?since_id=#{id1}&limit=1") + |> json_response_and_validate_schema(200) + + assert [%{"id" => ^id2}] = result end test "getting a list of blocks" do From 2b58b0dbce36042a8acbb41a8ddc8696d2e00d0e Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 13 Oct 2020 21:58:26 +0400 Subject: [PATCH 42/75] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fc1750d1..216d7bb32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`) - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`) - Mix task option for force-unfollowing relays +- Support pagination of blocks and mutes ### Changed From 1b8fd7e65af980c42b72f584c2a957b12ca5c78b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 16 Oct 2020 17:32:05 +0000 Subject: [PATCH 43/75] Adds feature to permit e.g., local admins and community moderators to automatically follow all newly registered accounts --- config/config.exs | 1 + config/description.exs | 11 +++++++++++ docs/configuration/cheatsheet.md | 1 + lib/pleroma/user.ex | 11 +++++++++++ test/pleroma/user_test.exs | 18 ++++++++++++++++++ 5 files changed, 42 insertions(+) diff --git a/config/config.exs b/config/config.exs index 2c6142360..0f3785d12 100644 --- a/config/config.exs +++ b/config/config.exs @@ -235,6 +235,7 @@ "text/bbcode" ], autofollowed_nicknames: [], + autofollowing_nicknames: [], max_pinned_statuses: 1, attachment_links: false, max_report_comment_size: 1000, diff --git a/config/description.exs b/config/description.exs index 2a1898922..2bbb12540 100644 --- a/config/description.exs +++ b/config/description.exs @@ -837,6 +837,17 @@ "rinpatch" ] }, + %{ + key: :autofollowing_nicknames, + type: {:list, :string}, + description: + "Set to nicknames of (local) users that automatically follows every newly registered user", + suggestions: [ + "admin", + "info", + "moderator", + ] + }, %{ key: :attachment_links, type: :boolean, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 0b13d7e88..f4b4b6c3c 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -45,6 +45,7 @@ To add configuration to your config file, you can copy it from the base config. older software for theses nicknames. * `max_pinned_statuses`: The maximum number of pinned statuses. `0` will disable the feature. * `autofollowed_nicknames`: Set to nicknames of (local) users that every new user should automatically follow. +* `autofollowing_nicknames`: Set to nicknames of (local) users that automatically follows every newly registered user. * `attachment_links`: Set to true to enable automatically adding attachment link text to statuses. * `max_report_comment_size`: The maximum size of the report comment (Default: `1000`). * `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). Default: `false`. diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index dc41d0001..2a3495103 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -765,6 +765,16 @@ defp autofollow_users(user) do follow_all(user, autofollowed_users) end + defp autofollowing_users(user) do + candidates = Config.get([:instance, :autofollowing_nicknames]) + + User.Query.build(%{nickname: candidates, local: true, deactivated: false}) + |> Repo.all() + |> Enum.each(&follow(&1, user, :follow_accept)) + + {:ok, :success} + end + @doc "Inserts provided changeset, performs post-registration actions (confirmation email sending etc.)" def register(%Ecto.Changeset{} = changeset) do with {:ok, user} <- Repo.insert(changeset) do @@ -774,6 +784,7 @@ def register(%Ecto.Changeset{} = changeset) do def post_register_action(%User{} = user) do with {:ok, user} <- autofollow_users(user), + {:ok, _} <- autofollowing_users(user), {:ok, user} <- set_cache(user), {:ok, _} <- send_welcome_email(user), {:ok, _} <- send_welcome_message(user), diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index 7220ce846..9ae52d594 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -388,6 +388,7 @@ test "fetches correct profile for nickname beginning with number" do } setup do: clear_config([:instance, :autofollowed_nicknames]) + setup do: clear_config([:instance, :autofollowing_nicknames]) setup do: clear_config([:welcome]) setup do: clear_config([:instance, :account_activation_required]) @@ -408,6 +409,23 @@ test "it autofollows accounts that are set for it" do refute User.following?(registered_user, remote_user) end + test "it adds automatic followers for new registered accounts" do + user1 = insert(:user) + user2 = insert(:user) + + Pleroma.Config.put([:instance, :autofollowing_nicknames], [ + user1.nickname, + user2.nickname + ]) + + cng = User.register_changeset(%User{}, @full_user_data) + + {:ok, registered_user} = User.register(cng) + + assert User.following?(user1, registered_user) + assert User.following?(user2, registered_user) + end + test "it sends a welcome message if it is set" do welcome_user = insert(:user) Pleroma.Config.put([:welcome, :direct_message, :enabled], true) From efd6572ffbf4cce86e28d351d3cbddd0a5334980 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 16 Oct 2020 17:43:44 +0000 Subject: [PATCH 44/75] Remove suggestions --- config/description.exs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/config/description.exs b/config/description.exs index 2bbb12540..79e1368d0 100644 --- a/config/description.exs +++ b/config/description.exs @@ -841,12 +841,7 @@ key: :autofollowing_nicknames, type: {:list, :string}, description: - "Set to nicknames of (local) users that automatically follows every newly registered user", - suggestions: [ - "admin", - "info", - "moderator", - ] + "Set to nicknames of (local) users that automatically follows every newly registered user" }, %{ key: :attachment_links, From d54233760f4c006d89aa80e0ae78cb6910fc74ab Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 17 Oct 2020 13:33:57 +0300 Subject: [PATCH 45/75] [#3053] Post-merge fix. --- lib/pleroma/web/feed/user_controller.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex index b66fdf275..a5013d2c0 100644 --- a/lib/pleroma/web/feed/user_controller.ex +++ b/lib/pleroma/web/feed/user_controller.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.Feed.UserController do use Pleroma.Web, :controller + alias Pleroma.Config alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.ActivityPubController From d16336e7fbf7851ee5f50a17747067f2be741581 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 17 Oct 2020 16:54:05 +0000 Subject: [PATCH 46/75] Document autofollowing_nicknames --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36a84b1a8..fd5b9034d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`) - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`) - Mix task option for force-unfollowing relays +- Setting `:instance, autofollowing_nicknames` to provide a way to make accounts automatically follow new users that register on the local Pleroma instance. ### Changed From 524fb0e4c2561f4a2e4c8e58519df991f034c901 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 18 Oct 2020 21:22:21 +0300 Subject: [PATCH 47/75] [#1668] Restricted access to app metrics endpoint by default. Added ability to configure IP whitelist for this endpoint. Added tests and documentation. --- CHANGELOG.md | 2 + config/config.exs | 7 +- docs/API/prometheus.md | 26 ++++++- lib/pleroma/helpers/inet_helper.ex | 19 +++++ lib/pleroma/web/endpoint.ex | 40 +++++++++-- .../web/endpoint/metrics_exporter_test.exs | 69 +++++++++++++++++++ 6 files changed, 154 insertions(+), 9 deletions(-) create mode 100644 lib/pleroma/helpers/inet_helper.ex create mode 100644 test/pleroma/web/endpoint/metrics_exporter_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 05e94581a..9f6a31f23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,12 +12,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details). - Pleroma API: Importing the mutes users from CSV files. - Experimental websocket-based federation between Pleroma instances. +- App metrics: ability to restrict access to specified IP whitelist. ### Changed - **Breaking** Requires `libmagic` (or `file`) to guess file types. - **Breaking:** Pleroma Admin API: emoji packs and files routes changed. - **Breaking:** Sensitive/NSFW statuses no longer disable link previews. +- **Breaking:** App metrics endpoint (`/api/pleroma/app_metrics`) is disabled by default, check `docs/API/prometheus.md` on enabling and configuring. - Search: Users are now findable by their urls. - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated. - Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated. diff --git a/config/config.exs b/config/config.exs index 2c6142360..a7aae5802 100644 --- a/config/config.exs +++ b/config/config.exs @@ -636,7 +636,12 @@ config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: false -config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, path: "/api/pleroma/app_metrics" +config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, + enabled: false, + auth: false, + ip_whitelist: [], + path: "/api/pleroma/app_metrics", + format: :text config :pleroma, Pleroma.ScheduledActivity, daily_user_limit: 25, diff --git a/docs/API/prometheus.md b/docs/API/prometheus.md index 19c564e3c..a5158d905 100644 --- a/docs/API/prometheus.md +++ b/docs/API/prometheus.md @@ -2,15 +2,37 @@ Pleroma includes support for exporting metrics via the [prometheus_ex](https://github.com/deadtrickster/prometheus.ex) library. +Config example: + +``` +config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, + enabled: true, + auth: {:basic, "myusername", "mypassword"}, + ip_whitelist: ["127.0.0.1"], + path: "/api/pleroma/app_metrics", + format: :text +``` + +* `enabled` (Pleroma extension) enables the endpoint +* `ip_whitelist` (Pleroma extension) could be used to restrict access only to specified IPs +* `auth` sets the authentication (`false` for no auth; configurable to HTTP Basic Auth, see [prometheus-plugs](https://github.com/deadtrickster/prometheus-plugs#exporting) documentation) +* `format` sets the output format (`:text` or `:protobuf`) +* `path` sets the path to app metrics page + + ## `/api/pleroma/app_metrics` + ### Exports Prometheus application metrics + * Method: `GET` -* Authentication: not required +* Authentication: not required by default (see configuration options above) * Params: none -* Response: JSON +* Response: text ## Grafana + ### Config example + The following is a config example to use with [Grafana](https://grafana.com) ``` diff --git a/lib/pleroma/helpers/inet_helper.ex b/lib/pleroma/helpers/inet_helper.ex new file mode 100644 index 000000000..126f82381 --- /dev/null +++ b/lib/pleroma/helpers/inet_helper.ex @@ -0,0 +1,19 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Helpers.InetHelper do + def parse_address(ip) when is_tuple(ip) do + {:ok, ip} + end + + def parse_address(ip) when is_binary(ip) do + ip + |> String.to_charlist() + |> parse_address() + end + + def parse_address(ip) do + :inet.parse_address(ip) + end +end diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 56562c12f..1a8fdd8b9 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -7,6 +7,8 @@ defmodule Pleroma.Web.Endpoint do require Pleroma.Constants + alias Pleroma.Config + socket("/socket", Pleroma.Web.UserSocket) plug(Pleroma.Web.Plugs.SetLocalePlug) @@ -86,19 +88,19 @@ defmodule Pleroma.Web.Endpoint do plug(Plug.Parsers, parsers: [ :urlencoded, - {:multipart, length: {Pleroma.Config, :get, [[:instance, :upload_limit]]}}, + {:multipart, length: {Config, :get, [[:instance, :upload_limit]]}}, :json ], pass: ["*/*"], json_decoder: Jason, - length: Pleroma.Config.get([:instance, :upload_limit]), + length: Config.get([:instance, :upload_limit]), body_reader: {Pleroma.Web.Plugs.DigestPlug, :read_body, []} ) plug(Plug.MethodOverride) plug(Plug.Head) - secure_cookies = Pleroma.Config.get([__MODULE__, :secure_cookie_flag]) + secure_cookies = Config.get([__MODULE__, :secure_cookie_flag]) cookie_name = if secure_cookies, @@ -106,7 +108,7 @@ defmodule Pleroma.Web.Endpoint do else: "pleroma_key" extra = - Pleroma.Config.get([__MODULE__, :extra_cookie_attrs]) + Config.get([__MODULE__, :extra_cookie_attrs]) |> Enum.join(";") # The session will be stored in the cookie and signed, @@ -116,7 +118,7 @@ defmodule Pleroma.Web.Endpoint do Plug.Session, store: :cookie, key: cookie_name, - signing_salt: Pleroma.Config.get([__MODULE__, :signing_salt], "CqaoopA2"), + signing_salt: Config.get([__MODULE__, :signing_salt], "CqaoopA2"), http_only: true, secure: secure_cookies, extra: extra @@ -136,8 +138,34 @@ defmodule MetricsExporter do use Prometheus.PlugExporter end + defmodule MetricsExporterCaller do + @behaviour Plug + + def init(opts), do: opts + + def call(conn, opts) do + prometheus_config = Application.get_env(:prometheus, MetricsExporter, []) + ip_whitelist = List.wrap(prometheus_config[:ip_whitelist]) + + cond do + !prometheus_config[:enabled] -> + conn + + ip_whitelist != [] and + !Enum.find(ip_whitelist, fn ip -> + Pleroma.Helpers.InetHelper.parse_address(ip) == {:ok, conn.remote_ip} + end) -> + conn + + true -> + MetricsExporter.call(conn, opts) + end + end + end + plug(PipelineInstrumenter) - plug(MetricsExporter) + + plug(MetricsExporterCaller) plug(Pleroma.Web.Router) diff --git a/test/pleroma/web/endpoint/metrics_exporter_test.exs b/test/pleroma/web/endpoint/metrics_exporter_test.exs new file mode 100644 index 000000000..f954cc1e7 --- /dev/null +++ b/test/pleroma/web/endpoint/metrics_exporter_test.exs @@ -0,0 +1,69 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Endpoint.MetricsExporterTest do + use Pleroma.Web.ConnCase + + alias Pleroma.Web.Endpoint.MetricsExporter + + defp config do + Application.get_env(:prometheus, MetricsExporter) + end + + describe "with default config" do + test "does NOT expose app metrics", %{conn: conn} do + conn + |> get(config()[:path]) + |> json_response(404) + end + end + + describe "when enabled" do + setup do + initial_config = config() + on_exit(fn -> Application.put_env(:prometheus, MetricsExporter, initial_config) end) + + Application.put_env( + :prometheus, + MetricsExporter, + Keyword.put(initial_config, :enabled, true) + ) + end + + test "serves app metrics", %{conn: conn} do + conn = get(conn, config()[:path]) + assert response = response(conn, 200) + + for metric <- [ + "http_requests_total", + "http_request_duration_microseconds", + "phoenix_controller_render_duration", + "phoenix_controller_call_duration", + "telemetry_scrape_duration", + "erlang_vm_memory_atom_bytes_total" + ] do + assert response =~ ~r/#{metric}/ + end + end + + test "when IP whitelist configured, " <> + "serves app metrics only if client IP is whitelisted", + %{conn: conn} do + Application.put_env( + :prometheus, + MetricsExporter, + Keyword.put(config(), :ip_whitelist, ["127.127.127.127", {1, 1, 1, 1}, '255.255.255.255']) + ) + + conn + |> get(config()[:path]) + |> json_response(404) + + conn + |> Map.put(:remote_ip, {127, 127, 127, 127}) + |> get(config()[:path]) + |> response(200) + end + end +end From 98f32cf8204113c6d019653c22e446e558147248 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 19 Oct 2020 15:30:32 +0400 Subject: [PATCH 48/75] Fix tests --- lib/pleroma/web/pleroma_api/controllers/backup_controller.ex | 2 +- test/backup_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex index e52c77ff2..8e3d081f3 100644 --- a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupController do use Pleroma.Web, :controller - alias Pleroma.Plugs.OAuthScopesPlug + alias Pleroma.Web.Plugs.OAuthScopesPlug action_fallback(Pleroma.Web.MastodonAPI.FallbackController) plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create]) diff --git a/test/backup_test.exs b/test/backup_test.exs index 23c08b680..078e03621 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -19,7 +19,7 @@ defmodule Pleroma.BackupTest do setup do clear_config([Pleroma.Upload, :uploader]) clear_config([Pleroma.Backup, :limit_days]) - clear_config([Pleroma.Emails.Mailer, :enabled]) + clear_config([Pleroma.Emails.Mailer, :enabled], true) end test "it requries enabled email" do From c1976d5b19fbceaecf1f52711fe35e1c7d5312aa Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 19 Oct 2020 18:14:39 +0400 Subject: [PATCH 49/75] Fix credo warnings --- test/{ => pleroma}/backup_test.exs | 0 .../web/pleroma_api/controllers/backup_controller_test.exs | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename test/{ => pleroma}/backup_test.exs (100%) rename test/{ => pleroma}/web/pleroma_api/controllers/backup_controller_test.exs (100%) diff --git a/test/backup_test.exs b/test/pleroma/backup_test.exs similarity index 100% rename from test/backup_test.exs rename to test/pleroma/backup_test.exs diff --git a/test/web/pleroma_api/controllers/backup_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs similarity index 100% rename from test/web/pleroma_api/controllers/backup_controller_test.exs rename to test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs From 3a28aa8814e186eea67df7da00df1636823291f3 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 20 Oct 2020 15:13:20 +0300 Subject: [PATCH 50/75] [#1668] Added :prometheus group config to config/description.exs. --- config/description.exs | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/config/description.exs b/config/description.exs index 2a1898922..0fe86ded7 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3722,5 +3722,41 @@ suggestions: [2] } ] + }, + %{ + group: :prometheus, + key: Pleroma.Web.Endpoint.MetricsExporter, + type: :group, + description: "Prometheus app metrics endpoint configuration", + children: [ + %{ + key: :enabled, + type: :boolean, + description: "[Pleroma extension] Enables app metrics endpoint." + }, + %{ + key: :ip_whitelist, + type: [{:list, :string}, {:list, :charlist}, {:list, :tuple}], + description: "[Pleroma extension] If non-empty, restricts access to app metrics endpoint to specified IP addresses." + }, + %{ + key: :auth, + type: [:boolean, :tuple], + description: "Enables HTTP Basic Auth for app metrics endpoint.", + suggestion: [false, {:basic, "myusername", "mypassword"}] + }, + %{ + key: :path, + type: :string, + description: "App metrics endpoint URI path.", + suggestions: ["/api/pleroma/app_metrics"] + }, + %{ + key: :format, + type: :atom, + description: "App metrics endpoint output format.", + suggestions: [:text, :protobuf] + } + ] } ] From ad605e3e16ba3f6ee3df7a0a3e6705036fef369f Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 20 Oct 2020 17:16:58 +0400 Subject: [PATCH 51/75] Rename `Pleroma.Backup` to `Pleroma.User.Backup` --- config/config.exs | 2 +- config/description.exs | 2 +- docs/configuration/cheatsheet.md | 2 +- lib/pleroma/{ => user}/backup.ex | 4 ++-- .../web/admin_api/controllers/admin_api_controller.ex | 2 +- .../web/pleroma_api/controllers/backup_controller.ex | 7 ++++--- lib/pleroma/web/pleroma_api/views/backup_view.ex | 2 +- lib/pleroma/workers/backup_worker.ex | 4 ++-- test/pleroma/{ => user}/backup_test.exs | 10 +++++----- .../controllers/admin_api_controller_test.exs | 6 +++--- .../pleroma_api/controllers/backup_controller_test.exs | 2 +- 11 files changed, 22 insertions(+), 21 deletions(-) rename lib/pleroma/{ => user}/backup.ex (98%) rename test/pleroma/{ => user}/backup_test.exs (97%) diff --git a/config/config.exs b/config/config.exs index 63e386250..c758c818c 100644 --- a/config/config.exs +++ b/config/config.exs @@ -831,7 +831,7 @@ config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator -config :pleroma, Pleroma.Backup, +config :pleroma, Pleroma.User.Backup, purge_after_days: 30, limit_days: 7, dir: nil diff --git a/config/description.exs b/config/description.exs index 88f2a6133..9f23b6d3d 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3731,7 +3731,7 @@ }, %{ group: :pleroma, - key: Pleroma.Backup, + key: Pleroma.User.Backup, type: :group, description: "Account Backup", children: [ diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index aafc43f3d..b40a2aebf 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -1077,7 +1077,7 @@ Control favicons for instances. * `enabled`: Allow/disallow displaying and getting instances favicons -## Account Backup +## Pleroma.User.Backup !!! note Requires enabled email diff --git a/lib/pleroma/backup.ex b/lib/pleroma/user/backup.ex similarity index 98% rename from lib/pleroma/backup.ex rename to lib/pleroma/user/backup.ex index 629e879a7..a9041fd94 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Backup do +defmodule Pleroma.User.Backup do use Ecto.Schema import Ecto.Changeset @@ -65,7 +65,7 @@ defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok defp validate_limit(user, nil) do case get_last(user.id) do %__MODULE__{inserted_at: inserted_at} -> - days = Pleroma.Config.get([Pleroma.Backup, :limit_days]) + days = Pleroma.Config.get([__MODULE__, :limit_days]) diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days) if diff > days do diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index a4f0d7d34..0a27c5861 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -685,7 +685,7 @@ def stats(conn, params) do def create_backup(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do with %User{} = user <- User.get_by_nickname(nickname), - {:ok, _} <- Pleroma.Backup.create(user, admin.id) do + {:ok, _} <- Pleroma.User.Backup.create(user, admin.id) do ModerationLog.insert_log(%{actor: admin, subject: user, action: "create_backup"}) json(conn, "") diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex index 8e3d081f3..bd7b36880 100644 --- a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupController do use Pleroma.Web, :controller alias Pleroma.Web.Plugs.OAuthScopesPlug + alias Pleroma.User.Backup action_fallback(Pleroma.Web.MastodonAPI.FallbackController) plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create]) @@ -14,13 +15,13 @@ defmodule Pleroma.Web.PleromaAPI.BackupController do defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaBackupOperation def index(%{assigns: %{user: user}} = conn, _params) do - backups = Pleroma.Backup.list(user) + backups = Backup.list(user) render(conn, "index.json", backups: backups) end def create(%{assigns: %{user: user}} = conn, _params) do - with {:ok, _} <- Pleroma.Backup.create(user) do - backups = Pleroma.Backup.list(user) + with {:ok, _} <- Backup.create(user) do + backups = Backup.list(user) render(conn, "index.json", backups: backups) end end diff --git a/lib/pleroma/web/pleroma_api/views/backup_view.ex b/lib/pleroma/web/pleroma_api/views/backup_view.ex index bf40a001e..af75876aa 100644 --- a/lib/pleroma/web/pleroma_api/views/backup_view.ex +++ b/lib/pleroma/web/pleroma_api/views/backup_view.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupView do use Pleroma.Web, :view - alias Pleroma.Backup + alias Pleroma.User.Backup alias Pleroma.Web.CommonAPI.Utils def render("show.json", %{backup: %Backup{} = backup}) do diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index 65754b6a2..5b4985983 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Workers.BackupWorker do use Oban.Worker, queue: :backup, max_attempts: 1 alias Oban.Job - alias Pleroma.Backup + alias Pleroma.User.Backup def process(backup, admin_user_id \\ nil) do %{"op" => "process", "backup_id" => backup.id, "admin_user_id" => admin_user_id} @@ -15,7 +15,7 @@ def process(backup, admin_user_id \\ nil) do end def schedule_deletion(backup) do - days = Pleroma.Config.get([Pleroma.Backup, :purge_after_days]) + days = Pleroma.Config.get([Backup, :purge_after_days]) time = 60 * 60 * 24 * days scheduled_at = Calendar.NaiveDateTime.add!(backup.inserted_at, time) diff --git a/test/pleroma/backup_test.exs b/test/pleroma/user/backup_test.exs similarity index 97% rename from test/pleroma/backup_test.exs rename to test/pleroma/user/backup_test.exs index 078e03621..5ad587833 100644 --- a/test/pleroma/backup_test.exs +++ b/test/pleroma/user/backup_test.exs @@ -2,7 +2,7 @@ # Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.BackupTest do +defmodule Pleroma.User.BackupTest do use Oban.Testing, repo: Pleroma.Repo use Pleroma.DataCase @@ -10,7 +10,7 @@ defmodule Pleroma.BackupTest do import Pleroma.Factory import Swoosh.TestAssertions - alias Pleroma.Backup + alias Pleroma.User.Backup alias Pleroma.Bookmark alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.CommonAPI @@ -18,7 +18,7 @@ defmodule Pleroma.BackupTest do setup do clear_config([Pleroma.Upload, :uploader]) - clear_config([Pleroma.Backup, :limit_days]) + clear_config([Backup, :limit_days]) clear_config([Pleroma.Emails.Mailer, :enabled], true) end @@ -44,7 +44,7 @@ test "it creates a backup record and an Oban job" do test "it return an error if the export limit is over" do %{id: user_id} = user = insert(:user) - limit_days = Pleroma.Config.get([Pleroma.Backup, :limit_days]) + limit_days = Pleroma.Config.get([Backup, :limit_days]) assert {:ok, %Oban.Job{args: args}} = Backup.create(user) backup = Backup.get(args["backup_id"]) assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup @@ -76,7 +76,7 @@ test "it process a backup record" do end test "it removes outdated backups after creating a fresh one" do - Pleroma.Config.put([Pleroma.Backup, :limit_days], -1) + Pleroma.Config.put([Backup, :limit_days], -1) Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) user = insert(:user) diff --git a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs index 34d48c2c1..5efe8ef71 100644 --- a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs @@ -2038,7 +2038,7 @@ test "it creates a backup", %{conn: conn} do |> post("/api/pleroma/admin/backups", %{nickname: user.nickname}) |> json_response(200) - assert [backup] = Repo.all(Pleroma.Backup) + assert [backup] = Repo.all(Pleroma.User.Backup) ObanHelpers.perform_all() @@ -2079,7 +2079,7 @@ test "it doesn't limit admins", %{conn: conn} do |> post("/api/pleroma/admin/backups", %{nickname: user.nickname}) |> json_response(200) - assert [_backup] = Repo.all(Pleroma.Backup) + assert [_backup] = Repo.all(Pleroma.User.Backup) assert "" == conn @@ -2088,7 +2088,7 @@ test "it doesn't limit admins", %{conn: conn} do |> post("/api/pleroma/admin/backups", %{nickname: user.nickname}) |> json_response(200) - assert Repo.aggregate(Pleroma.Backup, :count) == 2 + assert Repo.aggregate(Pleroma.User.Backup, :count) == 2 end end end diff --git a/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs index b2ac74c7d..f1941f6dd 100644 --- a/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs @@ -5,7 +5,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do use Pleroma.Web.ConnCase - alias Pleroma.Backup + alias Pleroma.User.Backup alias Pleroma.Web.PleromaAPI.BackupView setup do From 50d428088017e0383d8b35d4ab1b831f40646ab0 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 20 Oct 2020 16:18:24 +0300 Subject: [PATCH 52/75] [#1668] Formatting fix. --- config/description.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/description.exs b/config/description.exs index 0fe86ded7..11755d757 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3737,7 +3737,8 @@ %{ key: :ip_whitelist, type: [{:list, :string}, {:list, :charlist}, {:list, :tuple}], - description: "[Pleroma extension] If non-empty, restricts access to app metrics endpoint to specified IP addresses." + description: + "[Pleroma extension] If non-empty, restricts access to app metrics endpoint to specified IP addresses." }, %{ key: :auth, From 034ac43f3a91178694d3c621c52ce68207ec4f69 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 20 Oct 2020 17:47:04 +0400 Subject: [PATCH 53/75] Fix credo warnings --- lib/pleroma/web/pleroma_api/controllers/backup_controller.ex | 2 +- test/pleroma/user/backup_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex index bd7b36880..dd0a2e22f 100644 --- a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex @@ -5,8 +5,8 @@ defmodule Pleroma.Web.PleromaAPI.BackupController do use Pleroma.Web, :controller - alias Pleroma.Web.Plugs.OAuthScopesPlug alias Pleroma.User.Backup + alias Pleroma.Web.Plugs.OAuthScopesPlug action_fallback(Pleroma.Web.MastodonAPI.FallbackController) plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create]) diff --git a/test/pleroma/user/backup_test.exs b/test/pleroma/user/backup_test.exs index 5ad587833..513798911 100644 --- a/test/pleroma/user/backup_test.exs +++ b/test/pleroma/user/backup_test.exs @@ -10,9 +10,9 @@ defmodule Pleroma.User.BackupTest do import Pleroma.Factory import Swoosh.TestAssertions - alias Pleroma.User.Backup alias Pleroma.Bookmark alias Pleroma.Tests.ObanHelpers + alias Pleroma.User.Backup alias Pleroma.Web.CommonAPI alias Pleroma.Workers.BackupWorker From b18b93bbed925e5058d941662e93f0a46a27c325 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 21 Oct 2020 21:42:21 +0400 Subject: [PATCH 54/75] Fix formatting and typos in "Managing frontends" guide --- docs/administration/CLI_tasks/frontend.md | 93 +++++++++++++++-------- 1 file changed, 60 insertions(+), 33 deletions(-) diff --git a/docs/administration/CLI_tasks/frontend.md b/docs/administration/CLI_tasks/frontend.md index 7d1c1e937..d4a48cb56 100644 --- a/docs/administration/CLI_tasks/frontend.md +++ b/docs/administration/CLI_tasks/frontend.md @@ -1,12 +1,23 @@ # Managing frontends -`mix pleroma.frontend install [--ref ] [--file ] [--build-url ] [--path ] [--build-dir ]` +=== "OTP" + + ```sh + ./bin/pleroma_ctl frontend install [--ref ] [--file ] [--build-url ] [--path ] [--build-dir ] + ``` + +=== "From Source" + + ```sh + mix pleroma.frontend install [--ref ] [--file ] [--build-url ] [--path ] [--build-dir ] + ``` Frontend can be installed either from local zip file, or automatically downloaded from the web. -You can give all the options directly on the command like, but missing information will be filled out by looking at the data configured under `frontends.available` in the config files. +You can give all the options directly on the command line, but missing information will be filled out by looking at the data configured under `frontends.available` in the config files. + +Currently, known `` values are: -Currently known `` values are: - [admin-fe](https://git.pleroma.social/pleroma/admin-fe) - [kenoma](http://git.pleroma.social/lambadalambda/kenoma) - [pleroma-fe](http://git.pleroma.social/pleroma/pleroma-fe) @@ -19,51 +30,67 @@ You can still install frontends that are not configured, see below. For a frontend configured under the `available` key, it's enough to install it by name. -```sh tab="OTP" -./bin/pleroma_ctl frontend install pleroma -``` +=== "OTP" -```sh tab="From Source" -mix pleroma.frontend install pleroma -``` + ```sh + ./bin/pleroma_ctl frontend install pleroma + ``` -This will download the latest build for the the pre-configured `ref` and install it. It can then be configured as the one of the served frontends in the config file (see `primary` or `admin`). +=== "From Source" -You can override any of the details. To install a pleroma build from a different url, you could do this: + ```sh + mix pleroma.frontend install pleroma + ``` -```sh tab="OPT" -./bin/pleroma_ctl frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip -``` +This will download the latest build for the pre-configured `ref` and install it. It can then be configured as the one of the served frontends in the config file (see `primary` or `admin`). -```sh tab="From Source" -mix pleroma.frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip -``` +You can override any of the details. To install a pleroma build from a different URL, you could do this: + +=== "OTP" + + ```sh + ./bin/pleroma_ctl frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip + ``` + +=== "From Source" + + ```sh + mix pleroma.frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip + ``` Similarly, you can also install from a local zip file. -```sh tab="OTP" -./bin/pleroma_ctl frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip -``` +=== "OTP" -```sh tab="From Source" -mix pleroma.frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip -``` + ```sh + ./bin/pleroma_ctl frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip + ``` -The resulting frontend will always be installed into a folder of this template: `${instance_static}/frontends/${name}/${ref}` +=== "From Source" -Careful: This folder will be completely replaced on installation + ```sh + mix pleroma.frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip + ``` + +The resulting frontend will always be installed into a folder of this template: `${instance_static}/frontends/${name}/${ref}`. + +Careful: This folder will be completely replaced on installation. ## Example installation for an unknown frontend -The installation process is the same, but you will have to give all the needed options on the commond line. For example: +The installation process is the same, but you will have to give all the needed options on the command line. For example: -```sh tab="OTP" -./bin/pleroma_ctl frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip -``` +=== "OTP" -```sh tab="From Source" -mix pleroma.frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip -``` + ```sh + ./bin/pleroma_ctl frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip + ``` -If you don't have a zip file but just want to install a frontend from a local path, you can simply copy the files over a folder of this template: `${instance_static}/frontends/${name}/${ref}` +=== "From Source" + + ```sh + mix pleroma.frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip + ``` + +If you don't have a zip file but just want to install a frontend from a local path, you can simply copy the files over a folder of this template: `${instance_static}/frontends/${name}/${ref}`. From 9ef46ce4103f338d5bb75278013a63d3ae418d7e Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 21 Sep 2020 09:33:51 +0300 Subject: [PATCH 55/75] added 'unconfirmed' filter to admin/users --- lib/pleroma/user/query.ex | 5 + .../controllers/admin_api_controller.ex | 55 +-- .../controllers/admin_api_controller_test.exs | 399 +++++------------- test/pleroma/web/admin_api/search_test.exs | 11 + 4 files changed, 131 insertions(+), 339 deletions(-) diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex index 2440bf890..2933e7fb4 100644 --- a/lib/pleroma/user/query.ex +++ b/lib/pleroma/user/query.ex @@ -43,6 +43,7 @@ defmodule Pleroma.User.Query do active: boolean(), deactivated: boolean(), need_approval: boolean(), + need_confirmed: boolean(), is_admin: boolean(), is_moderator: boolean(), super_users: boolean(), @@ -156,6 +157,10 @@ defp compose_query({:need_approval, _}, query) do where(query, [u], u.approval_pending) end + defp compose_query({:need_confirmed, _}, query) do + where(query, [u], u.confirmation_pending) + end + defp compose_query({:followers, %User{id: id}}, query) do query |> where([u], u.id != ^id) diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index bdd3e195d..acfbeb0c8 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -5,7 +5,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do use Pleroma.Web, :controller - import Pleroma.Web.ControllerHelper, only: [json_response: 3] + import Pleroma.Web.ControllerHelper, + only: [json_response: 3, fetch_integer_param: 3] alias Pleroma.Config alias Pleroma.MFA @@ -100,12 +101,9 @@ def user_delete(conn, %{"nickname" => nickname}) do end def user_delete(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do - users = - nicknames - |> Enum.map(&User.get_cached_by_nickname/1) + users = Enum.map(nicknames, &User.get_cached_by_nickname/1) - users - |> Enum.each(fn user -> + Enum.each(users, fn user -> {:ok, delete_data, _} = Builder.delete(admin, user.ap_id) Pipeline.common_pipeline(delete_data, local: true) end) @@ -367,16 +365,18 @@ def list_users(conn, params) do {page, page_size} = page_params(params) filters = maybe_parse_filters(params["filters"]) - search_params = %{ - query: params["query"], - page: page, - page_size: page_size, - tags: params["tags"], - name: params["name"], - email: params["email"] - } + search_params = + %{ + query: params["query"], + page: page, + page_size: page_size, + tags: params["tags"], + name: params["name"], + email: params["email"] + } + |> Map.merge(filters) - with {:ok, users, count} <- Search.user(Map.merge(search_params, filters)) do + with {:ok, users, count} <- Search.user(search_params) do json( conn, AccountView.render("index.json", @@ -388,7 +388,7 @@ def list_users(conn, params) do end end - @filters ~w(local external active deactivated need_approval is_admin is_moderator) + @filters ~w(local external active deactivated need_approval need_confirmed is_admin is_moderator) @spec maybe_parse_filters(String.t()) :: %{required(String.t()) => true} | %{} defp maybe_parse_filters(filters) when is_nil(filters) or filters == "", do: %{} @@ -682,24 +682,9 @@ def stats(conn, params) do end defp page_params(params) do - {get_page(params["page"]), get_page_size(params["page_size"])} - end - - defp get_page(page_string) when is_nil(page_string), do: 1 - - defp get_page(page_string) do - case Integer.parse(page_string) do - {page, _} -> page - :error -> 1 - end - end - - defp get_page_size(page_size_string) when is_nil(page_size_string), do: @users_page_size - - defp get_page_size(page_size_string) do - case Integer.parse(page_size_string) do - {page_size, _} -> page_size - :error -> @users_page_size - end + { + fetch_integer_param(params, "page", 1), + fetch_integer_param(params, "page_size", @users_page_size) + } end end diff --git a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs index cba6b43d3..686b53a80 100644 --- a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs @@ -369,23 +369,7 @@ test "Show", %{conn: conn} do conn = get(conn, "/api/pleroma/admin/users/#{user.nickname}") - expected = %{ - "deactivated" => false, - "id" => to_string(user.id), - "local" => true, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "tags" => [], - "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => user.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } - - assert expected == json_response(conn, 200) + assert user_response(user) == json_response(conn, 200) end test "when the user doesn't exist", %{conn: conn} do @@ -652,51 +636,20 @@ test "renders users array for the first page", %{conn: conn, admin: admin} do users = [ - %{ - "deactivated" => admin.deactivated, - "id" => admin.id, - "nickname" => admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "local" => true, - "tags" => [], - "avatar" => User.avatar_url(admin) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(admin.name || admin.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => admin.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - }, - %{ - "deactivated" => user.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => false, - "tags" => ["foo", "bar"], - "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => user.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - }, - %{ - "deactivated" => user2.deactivated, - "id" => user2.id, - "nickname" => user2.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => true, - "tags" => [], - "avatar" => User.avatar_url(user2) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user2.name || user2.nickname), - "confirmation_pending" => false, - "approval_pending" => true, - "url" => user2.ap_id, - "registration_reason" => "I'm a chill dude", - "actor_type" => "Person" - } + user_response( + admin, + %{"roles" => %{"admin" => true, "moderator" => false}} + ), + user_response(user, %{"local" => false, "tags" => ["foo", "bar"]}), + user_response( + user2, + %{ + "local" => true, + "approval_pending" => true, + "registration_reason" => "I'm a chill dude", + "actor_type" => "Person" + } + ) ] |> Enum.sort_by(& &1["nickname"]) @@ -757,23 +710,7 @@ test "regular search", %{conn: conn} do assert json_response(conn, 200) == %{ "count" => 1, "page_size" => 50, - "users" => [ - %{ - "deactivated" => user.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => true, - "tags" => [], - "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => user.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } - ] + "users" => [user_response(user, %{"local" => true})] } end @@ -786,23 +723,7 @@ test "search by domain", %{conn: conn} do assert json_response(conn, 200) == %{ "count" => 1, "page_size" => 50, - "users" => [ - %{ - "deactivated" => user.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => true, - "tags" => [], - "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => user.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } - ] + "users" => [user_response(user)] } end @@ -815,23 +736,7 @@ test "search by full nickname", %{conn: conn} do assert json_response(conn, 200) == %{ "count" => 1, "page_size" => 50, - "users" => [ - %{ - "deactivated" => user.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => true, - "tags" => [], - "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => user.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } - ] + "users" => [user_response(user)] } end @@ -844,23 +749,7 @@ test "search by display name", %{conn: conn} do assert json_response(conn, 200) == %{ "count" => 1, "page_size" => 50, - "users" => [ - %{ - "deactivated" => user.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => true, - "tags" => [], - "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => user.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } - ] + "users" => [user_response(user)] } end @@ -873,23 +762,7 @@ test "search by email", %{conn: conn} do assert json_response(conn, 200) == %{ "count" => 1, "page_size" => 50, - "users" => [ - %{ - "deactivated" => user.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => true, - "tags" => [], - "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => user.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } - ] + "users" => [user_response(user)] } end @@ -902,23 +775,7 @@ test "regular search with page size", %{conn: conn} do assert json_response(conn1, 200) == %{ "count" => 2, "page_size" => 1, - "users" => [ - %{ - "deactivated" => user.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => true, - "tags" => [], - "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => user.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } - ] + "users" => [user_response(user)] } conn2 = get(conn, "/api/pleroma/admin/users?query=a&page_size=1&page=2") @@ -926,23 +783,7 @@ test "regular search with page size", %{conn: conn} do assert json_response(conn2, 200) == %{ "count" => 2, "page_size" => 1, - "users" => [ - %{ - "deactivated" => user2.deactivated, - "id" => user2.id, - "nickname" => user2.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => true, - "tags" => [], - "avatar" => User.avatar_url(user2) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user2.name || user2.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => user2.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } - ] + "users" => [user_response(user2)] } end @@ -962,23 +803,7 @@ test "only local users" do assert json_response(conn, 200) == %{ "count" => 1, "page_size" => 50, - "users" => [ - %{ - "deactivated" => user.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => true, - "tags" => [], - "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => user.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } - ] + "users" => [user_response(user)] } end @@ -992,51 +817,14 @@ test "only local users with no query", %{conn: conn, admin: old_admin} do users = [ - %{ - "deactivated" => user.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => true, - "tags" => [], - "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => user.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - }, - %{ - "deactivated" => admin.deactivated, - "id" => admin.id, - "nickname" => admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "local" => true, - "tags" => [], - "avatar" => User.avatar_url(admin) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(admin.name || admin.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => admin.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - }, - %{ + user_response(user), + user_response(admin, %{ + "roles" => %{"admin" => true, "moderator" => false} + }), + user_response(old_admin, %{ "deactivated" => false, - "id" => old_admin.id, - "local" => true, - "nickname" => old_admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "tags" => [], - "avatar" => User.avatar_url(old_admin) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(old_admin.name || old_admin.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => old_admin.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } + "roles" => %{"admin" => true, "moderator" => false} + }) ] |> Enum.sort_by(& &1["nickname"]) @@ -1047,6 +835,30 @@ test "only local users with no query", %{conn: conn, admin: old_admin} do } end + test "only unconfirmed users", %{conn: conn} do + sad_user = insert(:user, nickname: "sadboy", confirmation_pending: true) + old_user = insert(:user, nickname: "oldboy", confirmation_pending: true) + + insert(:user, nickname: "happyboy", approval_pending: false) + insert(:user, confirmation_pending: false) + + result = + conn + |> get("/api/pleroma/admin/users?filters=need_confirmed") + |> json_response(200) + + users = + Enum.map([old_user, sad_user], fn user -> + user_response(user, %{ + "confirmation_pending" => true, + "approval_pending" => false + }) + end) + |> Enum.sort_by(& &1["nickname"]) + + assert result == %{"count" => 2, "page_size" => 50, "users" => users} + end + test "only unapproved users", %{conn: conn} do user = insert(:user, @@ -1175,21 +987,22 @@ test "load users with tags list", %{conn: conn} do users = [ - %{ - "deactivated" => false, - "id" => user1.id, - "nickname" => user1.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => user1.local, - "tags" => ["first"], - "avatar" => User.avatar_url(user1) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user1.name || user1.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => user1.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - }, + user_response( + user1, + %{ + "deactivated" => false, + "roles" => %{"admin" => false, "moderator" => false}, + "local" => user1.local, + "tags" => ["first"], + "avatar" => User.avatar_url(user1) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user1.name || user1.nickname), + "confirmation_pending" => false, + "approval_pending" => false, + "url" => user1.ap_id, + "registration_reason" => nil, + "actor_type" => "Person" + } + ), %{ "deactivated" => false, "id" => user2.id, @@ -1253,23 +1066,7 @@ test "it works with multiple filters" do assert json_response(conn, 200) == %{ "count" => 1, "page_size" => 50, - "users" => [ - %{ - "deactivated" => user.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => user.local, - "tags" => [], - "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => user.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } - ] + "users" => [user_response(user)] } end @@ -1282,21 +1079,7 @@ test "it omits relay user", %{admin: admin, conn: conn} do "count" => 1, "page_size" => 50, "users" => [ - %{ - "deactivated" => admin.deactivated, - "id" => admin.id, - "nickname" => admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "local" => true, - "tags" => [], - "avatar" => User.avatar_url(admin) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(admin.name || admin.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => admin.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } + user_response(admin, %{"roles" => %{"admin" => true, "moderator" => false}}) ] } end @@ -1368,21 +1151,10 @@ test "PATCH /api/pleroma/admin/users/:nickname/toggle_activation", %{admin: admi conn = patch(conn, "/api/pleroma/admin/users/#{user.nickname}/toggle_activation") assert json_response(conn, 200) == - %{ - "deactivated" => !user.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => true, - "tags" => [], - "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => user.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } + user_response( + user, + %{"deactivated" => !user.deactivated} + ) log_entry = Repo.one(ModerationLog) @@ -2024,6 +1796,25 @@ test "by instance", %{conn: conn} do response["status_visibility"] end end + + defp user_response(user, attrs \\ %{}) do + %{ + "deactivated" => user.deactivated, + "id" => user.id, + "nickname" => user.nickname, + "roles" => %{"admin" => false, "moderator" => false}, + "local" => user.local, + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname), + "confirmation_pending" => false, + "approval_pending" => false, + "url" => user.ap_id, + "registration_reason" => nil, + "actor_type" => "Person" + } + |> Map.merge(attrs) + end end # Needed for testing diff --git a/test/pleroma/web/admin_api/search_test.exs b/test/pleroma/web/admin_api/search_test.exs index ceec64f1e..27ca396e6 100644 --- a/test/pleroma/web/admin_api/search_test.exs +++ b/test/pleroma/web/admin_api/search_test.exs @@ -178,6 +178,17 @@ test "it returns unapproved user" do assert count == 1 end + test "it returns unconfirmed user" do + unconfirmed = insert(:user, confirmation_pending: true) + insert(:user) + insert(:user) + + {:ok, _results, total} = Search.user() + {:ok, [^unconfirmed], count} = Search.user(%{need_confirmed: true}) + assert total == 3 + assert count == 1 + end + test "it returns non-discoverable users" do insert(:user) insert(:user, is_discoverable: false) From cf4f3937946eb0232be8bf8ddba08d365b4e17ee Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 21 Sep 2020 15:01:03 +0300 Subject: [PATCH 56/75] added AdminApi.UserController --- .../controllers/admin_api_controller.ex | 242 +---- .../admin_api/controllers/user_controller.ex | 280 +++++ lib/pleroma/web/router.ex | 22 +- .../controllers/admin_api_controller_test.exs | 837 --------------- .../controllers/user_controller_test.exs | 983 ++++++++++++++++++ 5 files changed, 1275 insertions(+), 1089 deletions(-) create mode 100644 lib/pleroma/web/admin_api/controllers/user_controller.ex create mode 100644 test/web/admin_api/controllers/user_controller_test.exs diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index acfbeb0c8..df5817cfa 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -14,12 +14,9 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do alias Pleroma.Stats alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.ActivityPub.Builder - alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.AdminAPI alias Pleroma.Web.AdminAPI.AccountView alias Pleroma.Web.AdminAPI.ModerationLogView - alias Pleroma.Web.AdminAPI.Search alias Pleroma.Web.Endpoint alias Pleroma.Web.Plugs.OAuthScopesPlug alias Pleroma.Web.Router @@ -29,7 +26,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do plug( OAuthScopesPlug, %{scopes: ["read:accounts"], admin: true} - when action in [:list_users, :user_show, :right_get, :show_user_credentials] + when action in [:right_get, :show_user_credentials] ) plug( @@ -38,12 +35,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do when action in [ :get_password_reset, :force_password_reset, - :user_delete, - :users_create, - :user_toggle_activation, - :user_activate, - :user_deactivate, - :user_approve, :tag_users, :untag_users, :right_add, @@ -55,12 +46,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do ] ) - plug( - OAuthScopesPlug, - %{scopes: ["write:follows"], admin: true} - when action in [:user_follow, :user_unfollow] - ) - plug( OAuthScopesPlug, %{scopes: ["read:statuses"], admin: true} @@ -96,129 +81,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do action_fallback(AdminAPI.FallbackController) - def user_delete(conn, %{"nickname" => nickname}) do - user_delete(conn, %{"nicknames" => [nickname]}) - end - - def user_delete(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do - users = Enum.map(nicknames, &User.get_cached_by_nickname/1) - - Enum.each(users, fn user -> - {:ok, delete_data, _} = Builder.delete(admin, user.ap_id) - Pipeline.common_pipeline(delete_data, local: true) - end) - - ModerationLog.insert_log(%{ - actor: admin, - subject: users, - action: "delete" - }) - - json(conn, nicknames) - end - - def user_follow(%{assigns: %{user: admin}} = conn, %{ - "follower" => follower_nick, - "followed" => followed_nick - }) do - with %User{} = follower <- User.get_cached_by_nickname(follower_nick), - %User{} = followed <- User.get_cached_by_nickname(followed_nick) do - User.follow(follower, followed) - - ModerationLog.insert_log(%{ - actor: admin, - followed: followed, - follower: follower, - action: "follow" - }) - end - - json(conn, "ok") - end - - def user_unfollow(%{assigns: %{user: admin}} = conn, %{ - "follower" => follower_nick, - "followed" => followed_nick - }) do - with %User{} = follower <- User.get_cached_by_nickname(follower_nick), - %User{} = followed <- User.get_cached_by_nickname(followed_nick) do - User.unfollow(follower, followed) - - ModerationLog.insert_log(%{ - actor: admin, - followed: followed, - follower: follower, - action: "unfollow" - }) - end - - json(conn, "ok") - end - - def users_create(%{assigns: %{user: admin}} = conn, %{"users" => users}) do - changesets = - Enum.map(users, fn %{"nickname" => nickname, "email" => email, "password" => password} -> - user_data = %{ - nickname: nickname, - name: nickname, - email: email, - password: password, - password_confirmation: password, - bio: "." - } - - User.register_changeset(%User{}, user_data, need_confirmation: false) - end) - |> Enum.reduce(Ecto.Multi.new(), fn changeset, multi -> - Ecto.Multi.insert(multi, Ecto.UUID.generate(), changeset) - end) - - case Pleroma.Repo.transaction(changesets) do - {:ok, users} -> - res = - users - |> Map.values() - |> Enum.map(fn user -> - {:ok, user} = User.post_register_action(user) - - user - end) - |> Enum.map(&AccountView.render("created.json", %{user: &1})) - - ModerationLog.insert_log(%{ - actor: admin, - subjects: Map.values(users), - action: "create" - }) - - json(conn, res) - - {:error, id, changeset, _} -> - res = - Enum.map(changesets.operations, fn - {current_id, {:changeset, _current_changeset, _}} when current_id == id -> - AccountView.render("create-error.json", %{changeset: changeset}) - - {_, {:changeset, current_changeset, _}} -> - AccountView.render("create-error.json", %{changeset: current_changeset}) - end) - - conn - |> put_status(:conflict) - |> json(res) - end - end - - def user_show(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do - with %User{} = user <- User.get_cached_by_nickname_or_id(nickname, for: admin) do - conn - |> put_view(AccountView) - |> render("show.json", %{user: user}) - else - _ -> {:error, :not_found} - end - end - def list_instance_statuses(conn, %{"instance" => instance} = params) do with_reblogs = params["with_reblogs"] == "true" || params["with_reblogs"] == true {page, page_size} = page_params(params) @@ -272,69 +134,6 @@ def list_user_chats(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname} end end - def user_toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do - user = User.get_cached_by_nickname(nickname) - - {:ok, updated_user} = User.deactivate(user, !user.deactivated) - - action = if user.deactivated, do: "activate", else: "deactivate" - - ModerationLog.insert_log(%{ - actor: admin, - subject: [user], - action: action - }) - - conn - |> put_view(AccountView) - |> render("show.json", %{user: updated_user}) - end - - def user_activate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do - users = Enum.map(nicknames, &User.get_cached_by_nickname/1) - {:ok, updated_users} = User.deactivate(users, false) - - ModerationLog.insert_log(%{ - actor: admin, - subject: users, - action: "activate" - }) - - conn - |> put_view(AccountView) - |> render("index.json", %{users: Keyword.values(updated_users)}) - end - - def user_deactivate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do - users = Enum.map(nicknames, &User.get_cached_by_nickname/1) - {:ok, updated_users} = User.deactivate(users, true) - - ModerationLog.insert_log(%{ - actor: admin, - subject: users, - action: "deactivate" - }) - - conn - |> put_view(AccountView) - |> render("index.json", %{users: Keyword.values(updated_users)}) - end - - def user_approve(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do - users = Enum.map(nicknames, &User.get_cached_by_nickname/1) - {:ok, updated_users} = User.approve(users) - - ModerationLog.insert_log(%{ - actor: admin, - subject: users, - action: "approve" - }) - - conn - |> put_view(AccountView) - |> render("index.json", %{users: updated_users}) - end - def tag_users(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames, "tags" => tags}) do with {:ok, _} <- User.tag(nicknames, tags) do ModerationLog.insert_log(%{ @@ -361,45 +160,6 @@ def untag_users(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames, " end end - def list_users(conn, params) do - {page, page_size} = page_params(params) - filters = maybe_parse_filters(params["filters"]) - - search_params = - %{ - query: params["query"], - page: page, - page_size: page_size, - tags: params["tags"], - name: params["name"], - email: params["email"] - } - |> Map.merge(filters) - - with {:ok, users, count} <- Search.user(search_params) do - json( - conn, - AccountView.render("index.json", - users: users, - count: count, - page_size: page_size - ) - ) - end - end - - @filters ~w(local external active deactivated need_approval need_confirmed is_admin is_moderator) - - @spec maybe_parse_filters(String.t()) :: %{required(String.t()) => true} | %{} - defp maybe_parse_filters(filters) when is_nil(filters) or filters == "", do: %{} - - defp maybe_parse_filters(filters) do - filters - |> String.split(",") - |> Enum.filter(&Enum.member?(@filters, &1)) - |> Map.new(&{String.to_existing_atom(&1), true}) - end - def right_add_multiple(%{assigns: %{user: admin}} = conn, %{ "permission_group" => permission_group, "nicknames" => nicknames diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex new file mode 100644 index 000000000..5e049aa0f --- /dev/null +++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex @@ -0,0 +1,280 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.UserController do + use Pleroma.Web, :controller + + import Pleroma.Web.ControllerHelper, + only: [fetch_integer_param: 3] + + alias Pleroma.ModerationLog + alias Pleroma.Plugs.OAuthScopesPlug + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Builder + alias Pleroma.Web.ActivityPub.Pipeline + alias Pleroma.Web.AdminAPI + alias Pleroma.Web.AdminAPI.AccountView + alias Pleroma.Web.AdminAPI.Search + + @users_page_size 50 + + plug( + OAuthScopesPlug, + %{scopes: ["read:accounts"], admin: true} + when action in [:list, :show] + ) + + plug( + OAuthScopesPlug, + %{scopes: ["write:accounts"], admin: true} + when action in [ + :delete, + :create, + :toggle_activation, + :activate, + :deactivate, + :approve + ] + ) + + plug( + OAuthScopesPlug, + %{scopes: ["write:follows"], admin: true} + when action in [:follow, :unfollow] + ) + + action_fallback(AdminAPI.FallbackController) + + def delete(conn, %{"nickname" => nickname}) do + delete(conn, %{"nicknames" => [nickname]}) + end + + def delete(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do + users = Enum.map(nicknames, &User.get_cached_by_nickname/1) + + Enum.each(users, fn user -> + {:ok, delete_data, _} = Builder.delete(admin, user.ap_id) + Pipeline.common_pipeline(delete_data, local: true) + end) + + ModerationLog.insert_log(%{ + actor: admin, + subject: users, + action: "delete" + }) + + json(conn, nicknames) + end + + def follow(%{assigns: %{user: admin}} = conn, %{ + "follower" => follower_nick, + "followed" => followed_nick + }) do + with %User{} = follower <- User.get_cached_by_nickname(follower_nick), + %User{} = followed <- User.get_cached_by_nickname(followed_nick) do + User.follow(follower, followed) + + ModerationLog.insert_log(%{ + actor: admin, + followed: followed, + follower: follower, + action: "follow" + }) + end + + json(conn, "ok") + end + + def unfollow(%{assigns: %{user: admin}} = conn, %{ + "follower" => follower_nick, + "followed" => followed_nick + }) do + with %User{} = follower <- User.get_cached_by_nickname(follower_nick), + %User{} = followed <- User.get_cached_by_nickname(followed_nick) do + User.unfollow(follower, followed) + + ModerationLog.insert_log(%{ + actor: admin, + followed: followed, + follower: follower, + action: "unfollow" + }) + end + + json(conn, "ok") + end + + def create(%{assigns: %{user: admin}} = conn, %{"users" => users}) do + changesets = + Enum.map(users, fn %{"nickname" => nickname, "email" => email, "password" => password} -> + user_data = %{ + nickname: nickname, + name: nickname, + email: email, + password: password, + password_confirmation: password, + bio: "." + } + + User.register_changeset(%User{}, user_data, need_confirmation: false) + end) + |> Enum.reduce(Ecto.Multi.new(), fn changeset, multi -> + Ecto.Multi.insert(multi, Ecto.UUID.generate(), changeset) + end) + + case Pleroma.Repo.transaction(changesets) do + {:ok, users} -> + res = + users + |> Map.values() + |> Enum.map(fn user -> + {:ok, user} = User.post_register_action(user) + + user + end) + |> Enum.map(&AccountView.render("created.json", %{user: &1})) + + ModerationLog.insert_log(%{ + actor: admin, + subjects: Map.values(users), + action: "create" + }) + + json(conn, res) + + {:error, id, changeset, _} -> + res = + Enum.map(changesets.operations, fn + {current_id, {:changeset, _current_changeset, _}} when current_id == id -> + AccountView.render("create-error.json", %{changeset: changeset}) + + {_, {:changeset, current_changeset, _}} -> + AccountView.render("create-error.json", %{changeset: current_changeset}) + end) + + conn + |> put_status(:conflict) + |> json(res) + end + end + + def show(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do + with %User{} = user <- User.get_cached_by_nickname_or_id(nickname, for: admin) do + conn + |> put_view(AccountView) + |> render("show.json", %{user: user}) + else + _ -> {:error, :not_found} + end + end + + def toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do + user = User.get_cached_by_nickname(nickname) + + {:ok, updated_user} = User.deactivate(user, !user.deactivated) + + action = if user.deactivated, do: "activate", else: "deactivate" + + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: action + }) + + conn + |> put_view(AccountView) + |> render("show.json", %{user: updated_user}) + end + + def activate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do + users = Enum.map(nicknames, &User.get_cached_by_nickname/1) + {:ok, updated_users} = User.deactivate(users, false) + + ModerationLog.insert_log(%{ + actor: admin, + subject: users, + action: "activate" + }) + + conn + |> put_view(AccountView) + |> render("index.json", %{users: Keyword.values(updated_users)}) + end + + def deactivate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do + users = Enum.map(nicknames, &User.get_cached_by_nickname/1) + {:ok, updated_users} = User.deactivate(users, true) + + ModerationLog.insert_log(%{ + actor: admin, + subject: users, + action: "deactivate" + }) + + conn + |> put_view(AccountView) + |> render("index.json", %{users: Keyword.values(updated_users)}) + end + + def approve(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do + users = Enum.map(nicknames, &User.get_cached_by_nickname/1) + {:ok, updated_users} = User.approve(users) + + ModerationLog.insert_log(%{ + actor: admin, + subject: users, + action: "approve" + }) + + conn + |> put_view(AccountView) + |> render("index.json", %{users: updated_users}) + end + + def list(conn, params) do + {page, page_size} = page_params(params) + filters = maybe_parse_filters(params["filters"]) + + search_params = + %{ + query: params["query"], + page: page, + page_size: page_size, + tags: params["tags"], + name: params["name"], + email: params["email"] + } + |> Map.merge(filters) + + with {:ok, users, count} <- Search.user(search_params) do + json( + conn, + AccountView.render("index.json", + users: users, + count: count, + page_size: page_size + ) + ) + end + end + + @filters ~w(local external active deactivated need_approval need_confirmed is_admin is_moderator) + + @spec maybe_parse_filters(String.t()) :: %{required(String.t()) => true} | %{} + defp maybe_parse_filters(filters) when is_nil(filters) or filters == "", do: %{} + + defp maybe_parse_filters(filters) do + filters + |> String.split(",") + |> Enum.filter(&Enum.member?(@filters, &1)) + |> Map.new(&{String.to_existing_atom(&1), true}) + end + + defp page_params(params) do + { + fetch_integer_param(params, "page", 1), + fetch_integer_param(params, "page_size", @users_page_size) + } + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index d2d939989..3a9605778 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -129,16 +129,7 @@ defmodule Pleroma.Web.Router do scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do pipe_through(:admin_api) - post("/users/follow", AdminAPIController, :user_follow) - post("/users/unfollow", AdminAPIController, :user_unfollow) - put("/users/disable_mfa", AdminAPIController, :disable_mfa) - delete("/users", AdminAPIController, :user_delete) - post("/users", AdminAPIController, :users_create) - patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation) - patch("/users/activate", AdminAPIController, :user_activate) - patch("/users/deactivate", AdminAPIController, :user_deactivate) - patch("/users/approve", AdminAPIController, :user_approve) put("/users/tag", AdminAPIController, :tag_users) delete("/users/tag", AdminAPIController, :untag_users) @@ -161,6 +152,15 @@ defmodule Pleroma.Web.Router do :right_delete_multiple ) + post("/users/follow", UserController, :follow) + post("/users/unfollow", UserController, :unfollow) + delete("/users", UserController, :delete) + post("/users", UserController, :create) + patch("/users/:nickname/toggle_activation", UserController, :toggle_activation) + patch("/users/activate", UserController, :activate) + patch("/users/deactivate", UserController, :deactivate) + patch("/users/approve", UserController, :approve) + get("/relay", RelayController, :index) post("/relay", RelayController, :follow) delete("/relay", RelayController, :unfollow) @@ -175,8 +175,8 @@ defmodule Pleroma.Web.Router do get("/users/:nickname/credentials", AdminAPIController, :show_user_credentials) patch("/users/:nickname/credentials", AdminAPIController, :update_user_credentials) - get("/users", AdminAPIController, :list_users) - get("/users/:nickname", AdminAPIController, :user_show) + get("/users", UserController, :list) + get("/users/:nickname", UserController, :show) get("/users/:nickname/statuses", AdminAPIController, :list_user_statuses) get("/users/:nickname/chats", AdminAPIController, :list_user_chats) diff --git a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs index 686b53a80..34b26dddf 100644 --- a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs @@ -7,22 +7,17 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do use Oban.Testing, repo: Pleroma.Repo import ExUnit.CaptureLog - import Mock import Pleroma.Factory import Swoosh.TestAssertions alias Pleroma.Activity alias Pleroma.Config - alias Pleroma.HTML alias Pleroma.MFA alias Pleroma.ModerationLog alias Pleroma.Repo alias Pleroma.Tests.ObanHelpers alias Pleroma.User - alias Pleroma.Web - alias Pleroma.Web.ActivityPub.Relay alias Pleroma.Web.CommonAPI - alias Pleroma.Web.MediaProxy setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -153,284 +148,6 @@ test "GET /api/pleroma/admin/users/:nickname requires " <> end end - describe "DELETE /api/pleroma/admin/users" do - test "single user", %{admin: admin, conn: conn} do - clear_config([:instance, :federating], true) - - user = - insert(:user, - avatar: %{"url" => [%{"href" => "https://someurl"}]}, - banner: %{"url" => [%{"href" => "https://somebanner"}]}, - bio: "Hello world!", - name: "A guy" - ) - - # Create some activities to check they got deleted later - follower = insert(:user) - {:ok, _} = CommonAPI.post(user, %{status: "test"}) - {:ok, _, _, _} = CommonAPI.follow(user, follower) - {:ok, _, _, _} = CommonAPI.follow(follower, user) - user = Repo.get(User, user.id) - assert user.note_count == 1 - assert user.follower_count == 1 - assert user.following_count == 1 - refute user.deactivated - - with_mock Pleroma.Web.Federator, - publish: fn _ -> nil end, - perform: fn _, _ -> nil end do - conn = - conn - |> put_req_header("accept", "application/json") - |> delete("/api/pleroma/admin/users?nickname=#{user.nickname}") - - ObanHelpers.perform_all() - - assert User.get_by_nickname(user.nickname).deactivated - - log_entry = Repo.one(ModerationLog) - - assert ModerationLog.get_log_entry_message(log_entry) == - "@#{admin.nickname} deleted users: @#{user.nickname}" - - assert json_response(conn, 200) == [user.nickname] - - user = Repo.get(User, user.id) - assert user.deactivated - - assert user.avatar == %{} - assert user.banner == %{} - assert user.note_count == 0 - assert user.follower_count == 0 - assert user.following_count == 0 - assert user.bio == "" - assert user.name == nil - - assert called(Pleroma.Web.Federator.publish(:_)) - end - end - - test "multiple users", %{admin: admin, conn: conn} do - user_one = insert(:user) - user_two = insert(:user) - - conn = - conn - |> put_req_header("accept", "application/json") - |> delete("/api/pleroma/admin/users", %{ - nicknames: [user_one.nickname, user_two.nickname] - }) - - log_entry = Repo.one(ModerationLog) - - assert ModerationLog.get_log_entry_message(log_entry) == - "@#{admin.nickname} deleted users: @#{user_one.nickname}, @#{user_two.nickname}" - - response = json_response(conn, 200) - assert response -- [user_one.nickname, user_two.nickname] == [] - end - end - - describe "/api/pleroma/admin/users" do - test "Create", %{conn: conn} do - conn = - conn - |> put_req_header("accept", "application/json") - |> post("/api/pleroma/admin/users", %{ - "users" => [ - %{ - "nickname" => "lain", - "email" => "lain@example.org", - "password" => "test" - }, - %{ - "nickname" => "lain2", - "email" => "lain2@example.org", - "password" => "test" - } - ] - }) - - response = json_response(conn, 200) |> Enum.map(&Map.get(&1, "type")) - assert response == ["success", "success"] - - log_entry = Repo.one(ModerationLog) - - assert ["lain", "lain2"] -- Enum.map(log_entry.data["subjects"], & &1["nickname"]) == [] - end - - test "Cannot create user with existing email", %{conn: conn} do - user = insert(:user) - - conn = - conn - |> put_req_header("accept", "application/json") - |> post("/api/pleroma/admin/users", %{ - "users" => [ - %{ - "nickname" => "lain", - "email" => user.email, - "password" => "test" - } - ] - }) - - assert json_response(conn, 409) == [ - %{ - "code" => 409, - "data" => %{ - "email" => user.email, - "nickname" => "lain" - }, - "error" => "email has already been taken", - "type" => "error" - } - ] - end - - test "Cannot create user with existing nickname", %{conn: conn} do - user = insert(:user) - - conn = - conn - |> put_req_header("accept", "application/json") - |> post("/api/pleroma/admin/users", %{ - "users" => [ - %{ - "nickname" => user.nickname, - "email" => "someuser@plerama.social", - "password" => "test" - } - ] - }) - - assert json_response(conn, 409) == [ - %{ - "code" => 409, - "data" => %{ - "email" => "someuser@plerama.social", - "nickname" => user.nickname - }, - "error" => "nickname has already been taken", - "type" => "error" - } - ] - end - - test "Multiple user creation works in transaction", %{conn: conn} do - user = insert(:user) - - conn = - conn - |> put_req_header("accept", "application/json") - |> post("/api/pleroma/admin/users", %{ - "users" => [ - %{ - "nickname" => "newuser", - "email" => "newuser@pleroma.social", - "password" => "test" - }, - %{ - "nickname" => "lain", - "email" => user.email, - "password" => "test" - } - ] - }) - - assert json_response(conn, 409) == [ - %{ - "code" => 409, - "data" => %{ - "email" => user.email, - "nickname" => "lain" - }, - "error" => "email has already been taken", - "type" => "error" - }, - %{ - "code" => 409, - "data" => %{ - "email" => "newuser@pleroma.social", - "nickname" => "newuser" - }, - "error" => "", - "type" => "error" - } - ] - - assert User.get_by_nickname("newuser") === nil - end - end - - describe "/api/pleroma/admin/users/:nickname" do - test "Show", %{conn: conn} do - user = insert(:user) - - conn = get(conn, "/api/pleroma/admin/users/#{user.nickname}") - - assert user_response(user) == json_response(conn, 200) - end - - test "when the user doesn't exist", %{conn: conn} do - user = build(:user) - - conn = get(conn, "/api/pleroma/admin/users/#{user.nickname}") - - assert %{"error" => "Not found"} == json_response(conn, 404) - end - end - - describe "/api/pleroma/admin/users/follow" do - test "allows to force-follow another user", %{admin: admin, conn: conn} do - user = insert(:user) - follower = insert(:user) - - conn - |> put_req_header("accept", "application/json") - |> post("/api/pleroma/admin/users/follow", %{ - "follower" => follower.nickname, - "followed" => user.nickname - }) - - user = User.get_cached_by_id(user.id) - follower = User.get_cached_by_id(follower.id) - - assert User.following?(follower, user) - - log_entry = Repo.one(ModerationLog) - - assert ModerationLog.get_log_entry_message(log_entry) == - "@#{admin.nickname} made @#{follower.nickname} follow @#{user.nickname}" - end - end - - describe "/api/pleroma/admin/users/unfollow" do - test "allows to force-unfollow another user", %{admin: admin, conn: conn} do - user = insert(:user) - follower = insert(:user) - - User.follow(follower, user) - - conn - |> put_req_header("accept", "application/json") - |> post("/api/pleroma/admin/users/unfollow", %{ - "follower" => follower.nickname, - "followed" => user.nickname - }) - - user = User.get_cached_by_id(user.id) - follower = User.get_cached_by_id(follower.id) - - refute User.following?(follower, user) - - log_entry = Repo.one(ModerationLog) - - assert ModerationLog.get_log_entry_message(log_entry) == - "@#{admin.nickname} made @#{follower.nickname} unfollow @#{user.nickname}" - end - end - describe "PUT /api/pleroma/admin/users/tag" do setup %{conn: conn} do user1 = insert(:user, %{tags: ["x"]}) @@ -627,541 +344,6 @@ test "/api/pleroma/admin/users/:nickname/password_reset", %{conn: conn} do assert Regex.match?(~r/(http:\/\/|https:\/\/)/, resp["link"]) end - describe "GET /api/pleroma/admin/users" do - test "renders users array for the first page", %{conn: conn, admin: admin} do - user = insert(:user, local: false, tags: ["foo", "bar"]) - user2 = insert(:user, approval_pending: true, registration_reason: "I'm a chill dude") - - conn = get(conn, "/api/pleroma/admin/users?page=1") - - users = - [ - user_response( - admin, - %{"roles" => %{"admin" => true, "moderator" => false}} - ), - user_response(user, %{"local" => false, "tags" => ["foo", "bar"]}), - user_response( - user2, - %{ - "local" => true, - "approval_pending" => true, - "registration_reason" => "I'm a chill dude", - "actor_type" => "Person" - } - ) - ] - |> Enum.sort_by(& &1["nickname"]) - - assert json_response(conn, 200) == %{ - "count" => 3, - "page_size" => 50, - "users" => users - } - end - - test "pagination works correctly with service users", %{conn: conn} do - service1 = User.get_or_create_service_actor_by_ap_id(Web.base_url() <> "/meido", "meido") - - insert_list(25, :user) - - assert %{"count" => 26, "page_size" => 10, "users" => users1} = - conn - |> get("/api/pleroma/admin/users?page=1&filters=", %{page_size: "10"}) - |> json_response(200) - - assert Enum.count(users1) == 10 - assert service1 not in users1 - - assert %{"count" => 26, "page_size" => 10, "users" => users2} = - conn - |> get("/api/pleroma/admin/users?page=2&filters=", %{page_size: "10"}) - |> json_response(200) - - assert Enum.count(users2) == 10 - assert service1 not in users2 - - assert %{"count" => 26, "page_size" => 10, "users" => users3} = - conn - |> get("/api/pleroma/admin/users?page=3&filters=", %{page_size: "10"}) - |> json_response(200) - - assert Enum.count(users3) == 6 - assert service1 not in users3 - end - - test "renders empty array for the second page", %{conn: conn} do - insert(:user) - - conn = get(conn, "/api/pleroma/admin/users?page=2") - - assert json_response(conn, 200) == %{ - "count" => 2, - "page_size" => 50, - "users" => [] - } - end - - test "regular search", %{conn: conn} do - user = insert(:user, nickname: "bob") - - conn = get(conn, "/api/pleroma/admin/users?query=bo") - - assert json_response(conn, 200) == %{ - "count" => 1, - "page_size" => 50, - "users" => [user_response(user, %{"local" => true})] - } - end - - test "search by domain", %{conn: conn} do - user = insert(:user, nickname: "nickname@domain.com") - insert(:user) - - conn = get(conn, "/api/pleroma/admin/users?query=domain.com") - - assert json_response(conn, 200) == %{ - "count" => 1, - "page_size" => 50, - "users" => [user_response(user)] - } - end - - test "search by full nickname", %{conn: conn} do - user = insert(:user, nickname: "nickname@domain.com") - insert(:user) - - conn = get(conn, "/api/pleroma/admin/users?query=nickname@domain.com") - - assert json_response(conn, 200) == %{ - "count" => 1, - "page_size" => 50, - "users" => [user_response(user)] - } - end - - test "search by display name", %{conn: conn} do - user = insert(:user, name: "Display name") - insert(:user) - - conn = get(conn, "/api/pleroma/admin/users?name=display") - - assert json_response(conn, 200) == %{ - "count" => 1, - "page_size" => 50, - "users" => [user_response(user)] - } - end - - test "search by email", %{conn: conn} do - user = insert(:user, email: "email@example.com") - insert(:user) - - conn = get(conn, "/api/pleroma/admin/users?email=email@example.com") - - assert json_response(conn, 200) == %{ - "count" => 1, - "page_size" => 50, - "users" => [user_response(user)] - } - end - - test "regular search with page size", %{conn: conn} do - user = insert(:user, nickname: "aalice") - user2 = insert(:user, nickname: "alice") - - conn1 = get(conn, "/api/pleroma/admin/users?query=a&page_size=1&page=1") - - assert json_response(conn1, 200) == %{ - "count" => 2, - "page_size" => 1, - "users" => [user_response(user)] - } - - conn2 = get(conn, "/api/pleroma/admin/users?query=a&page_size=1&page=2") - - assert json_response(conn2, 200) == %{ - "count" => 2, - "page_size" => 1, - "users" => [user_response(user2)] - } - end - - test "only local users" do - admin = insert(:user, is_admin: true, nickname: "john") - token = insert(:oauth_admin_token, user: admin) - user = insert(:user, nickname: "bob") - - insert(:user, nickname: "bobb", local: false) - - conn = - build_conn() - |> assign(:user, admin) - |> assign(:token, token) - |> get("/api/pleroma/admin/users?query=bo&filters=local") - - assert json_response(conn, 200) == %{ - "count" => 1, - "page_size" => 50, - "users" => [user_response(user)] - } - end - - test "only local users with no query", %{conn: conn, admin: old_admin} do - admin = insert(:user, is_admin: true, nickname: "john") - user = insert(:user, nickname: "bob") - - insert(:user, nickname: "bobb", local: false) - - conn = get(conn, "/api/pleroma/admin/users?filters=local") - - users = - [ - user_response(user), - user_response(admin, %{ - "roles" => %{"admin" => true, "moderator" => false} - }), - user_response(old_admin, %{ - "deactivated" => false, - "roles" => %{"admin" => true, "moderator" => false} - }) - ] - |> Enum.sort_by(& &1["nickname"]) - - assert json_response(conn, 200) == %{ - "count" => 3, - "page_size" => 50, - "users" => users - } - end - - test "only unconfirmed users", %{conn: conn} do - sad_user = insert(:user, nickname: "sadboy", confirmation_pending: true) - old_user = insert(:user, nickname: "oldboy", confirmation_pending: true) - - insert(:user, nickname: "happyboy", approval_pending: false) - insert(:user, confirmation_pending: false) - - result = - conn - |> get("/api/pleroma/admin/users?filters=need_confirmed") - |> json_response(200) - - users = - Enum.map([old_user, sad_user], fn user -> - user_response(user, %{ - "confirmation_pending" => true, - "approval_pending" => false - }) - end) - |> Enum.sort_by(& &1["nickname"]) - - assert result == %{"count" => 2, "page_size" => 50, "users" => users} - end - - test "only unapproved users", %{conn: conn} do - user = - insert(:user, - nickname: "sadboy", - approval_pending: true, - registration_reason: "Plz let me in!" - ) - - insert(:user, nickname: "happyboy", approval_pending: false) - - conn = get(conn, "/api/pleroma/admin/users?filters=need_approval") - - users = - [ - %{ - "deactivated" => user.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => true, - "tags" => [], - "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname), - "confirmation_pending" => false, - "approval_pending" => true, - "url" => user.ap_id, - "registration_reason" => "Plz let me in!", - "actor_type" => "Person" - } - ] - |> Enum.sort_by(& &1["nickname"]) - - assert json_response(conn, 200) == %{ - "count" => 1, - "page_size" => 50, - "users" => users - } - end - - test "load only admins", %{conn: conn, admin: admin} do - second_admin = insert(:user, is_admin: true) - insert(:user) - insert(:user) - - conn = get(conn, "/api/pleroma/admin/users?filters=is_admin") - - users = - [ - %{ - "deactivated" => false, - "id" => admin.id, - "nickname" => admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "local" => admin.local, - "tags" => [], - "avatar" => User.avatar_url(admin) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(admin.name || admin.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => admin.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - }, - %{ - "deactivated" => false, - "id" => second_admin.id, - "nickname" => second_admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "local" => second_admin.local, - "tags" => [], - "avatar" => User.avatar_url(second_admin) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(second_admin.name || second_admin.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => second_admin.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } - ] - |> Enum.sort_by(& &1["nickname"]) - - assert json_response(conn, 200) == %{ - "count" => 2, - "page_size" => 50, - "users" => users - } - end - - test "load only moderators", %{conn: conn} do - moderator = insert(:user, is_moderator: true) - insert(:user) - insert(:user) - - conn = get(conn, "/api/pleroma/admin/users?filters=is_moderator") - - assert json_response(conn, 200) == %{ - "count" => 1, - "page_size" => 50, - "users" => [ - %{ - "deactivated" => false, - "id" => moderator.id, - "nickname" => moderator.nickname, - "roles" => %{"admin" => false, "moderator" => true}, - "local" => moderator.local, - "tags" => [], - "avatar" => User.avatar_url(moderator) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(moderator.name || moderator.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => moderator.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } - ] - } - end - - test "load users with tags list", %{conn: conn} do - user1 = insert(:user, tags: ["first"]) - user2 = insert(:user, tags: ["second"]) - insert(:user) - insert(:user) - - conn = get(conn, "/api/pleroma/admin/users?tags[]=first&tags[]=second") - - users = - [ - user_response( - user1, - %{ - "deactivated" => false, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => user1.local, - "tags" => ["first"], - "avatar" => User.avatar_url(user1) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user1.name || user1.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => user1.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } - ), - %{ - "deactivated" => false, - "id" => user2.id, - "nickname" => user2.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => user2.local, - "tags" => ["second"], - "avatar" => User.avatar_url(user2) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user2.name || user2.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => user2.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } - ] - |> Enum.sort_by(& &1["nickname"]) - - assert json_response(conn, 200) == %{ - "count" => 2, - "page_size" => 50, - "users" => users - } - end - - test "`active` filters out users pending approval", %{token: token} do - insert(:user, approval_pending: true) - %{id: user_id} = insert(:user, approval_pending: false) - %{id: admin_id} = token.user - - conn = - build_conn() - |> assign(:user, token.user) - |> assign(:token, token) - |> get("/api/pleroma/admin/users?filters=active") - - assert %{ - "count" => 2, - "page_size" => 50, - "users" => [ - %{"id" => ^admin_id}, - %{"id" => ^user_id} - ] - } = json_response(conn, 200) - end - - test "it works with multiple filters" do - admin = insert(:user, nickname: "john", is_admin: true) - token = insert(:oauth_admin_token, user: admin) - user = insert(:user, nickname: "bob", local: false, deactivated: true) - - insert(:user, nickname: "ken", local: true, deactivated: true) - insert(:user, nickname: "bobb", local: false, deactivated: false) - - conn = - build_conn() - |> assign(:user, admin) - |> assign(:token, token) - |> get("/api/pleroma/admin/users?filters=deactivated,external") - - assert json_response(conn, 200) == %{ - "count" => 1, - "page_size" => 50, - "users" => [user_response(user)] - } - end - - test "it omits relay user", %{admin: admin, conn: conn} do - assert %User{} = Relay.get_actor() - - conn = get(conn, "/api/pleroma/admin/users") - - assert json_response(conn, 200) == %{ - "count" => 1, - "page_size" => 50, - "users" => [ - user_response(admin, %{"roles" => %{"admin" => true, "moderator" => false}}) - ] - } - end - end - - test "PATCH /api/pleroma/admin/users/activate", %{admin: admin, conn: conn} do - user_one = insert(:user, deactivated: true) - user_two = insert(:user, deactivated: true) - - conn = - patch( - conn, - "/api/pleroma/admin/users/activate", - %{nicknames: [user_one.nickname, user_two.nickname]} - ) - - response = json_response(conn, 200) - assert Enum.map(response["users"], & &1["deactivated"]) == [false, false] - - log_entry = Repo.one(ModerationLog) - - assert ModerationLog.get_log_entry_message(log_entry) == - "@#{admin.nickname} activated users: @#{user_one.nickname}, @#{user_two.nickname}" - end - - test "PATCH /api/pleroma/admin/users/deactivate", %{admin: admin, conn: conn} do - user_one = insert(:user, deactivated: false) - user_two = insert(:user, deactivated: false) - - conn = - patch( - conn, - "/api/pleroma/admin/users/deactivate", - %{nicknames: [user_one.nickname, user_two.nickname]} - ) - - response = json_response(conn, 200) - assert Enum.map(response["users"], & &1["deactivated"]) == [true, true] - - log_entry = Repo.one(ModerationLog) - - assert ModerationLog.get_log_entry_message(log_entry) == - "@#{admin.nickname} deactivated users: @#{user_one.nickname}, @#{user_two.nickname}" - end - - test "PATCH /api/pleroma/admin/users/approve", %{admin: admin, conn: conn} do - user_one = insert(:user, approval_pending: true) - user_two = insert(:user, approval_pending: true) - - conn = - patch( - conn, - "/api/pleroma/admin/users/approve", - %{nicknames: [user_one.nickname, user_two.nickname]} - ) - - response = json_response(conn, 200) - assert Enum.map(response["users"], & &1["approval_pending"]) == [false, false] - - log_entry = Repo.one(ModerationLog) - - assert ModerationLog.get_log_entry_message(log_entry) == - "@#{admin.nickname} approved users: @#{user_one.nickname}, @#{user_two.nickname}" - end - - test "PATCH /api/pleroma/admin/users/:nickname/toggle_activation", %{admin: admin, conn: conn} do - user = insert(:user) - - conn = patch(conn, "/api/pleroma/admin/users/#{user.nickname}/toggle_activation") - - assert json_response(conn, 200) == - user_response( - user, - %{"deactivated" => !user.deactivated} - ) - - log_entry = Repo.one(ModerationLog) - - assert ModerationLog.get_log_entry_message(log_entry) == - "@#{admin.nickname} deactivated users: @#{user.nickname}" - end - describe "PUT disable_mfa" do test "returns 200 and disable 2fa", %{conn: conn} do user = @@ -1796,25 +978,6 @@ test "by instance", %{conn: conn} do response["status_visibility"] end end - - defp user_response(user, attrs \\ %{}) do - %{ - "deactivated" => user.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => user.local, - "tags" => [], - "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => user.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } - |> Map.merge(attrs) - end end # Needed for testing diff --git a/test/web/admin_api/controllers/user_controller_test.exs b/test/web/admin_api/controllers/user_controller_test.exs new file mode 100644 index 000000000..347258058 --- /dev/null +++ b/test/web/admin_api/controllers/user_controller_test.exs @@ -0,0 +1,983 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.UserControllerTest do + use Pleroma.Web.ConnCase + use Oban.Testing, repo: Pleroma.Repo + + import Mock + import Pleroma.Factory + + alias Pleroma.Config + alias Pleroma.HTML + alias Pleroma.ModerationLog + alias Pleroma.Repo + alias Pleroma.Tests.ObanHelpers + alias Pleroma.User + alias Pleroma.Web + alias Pleroma.Web.ActivityPub.Relay + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.MediaProxy + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + + :ok + end + + setup do + admin = insert(:user, is_admin: true) + token = insert(:oauth_admin_token, user: admin) + + conn = + build_conn() + |> assign(:user, admin) + |> assign(:token, token) + + {:ok, %{admin: admin, token: token, conn: conn}} + end + + test "with valid `admin_token` query parameter, skips OAuth scopes check" do + clear_config([:admin_token], "password123") + + user = insert(:user) + + conn = get(build_conn(), "/api/pleroma/admin/users/#{user.nickname}?admin_token=password123") + + assert json_response(conn, 200) + end + + describe "with [:auth, :enforce_oauth_admin_scope_usage]," do + setup do: clear_config([:auth, :enforce_oauth_admin_scope_usage], true) + + test "GET /api/pleroma/admin/users/:nickname requires admin:read:accounts or broader scope", + %{admin: admin} do + user = insert(:user) + url = "/api/pleroma/admin/users/#{user.nickname}" + + good_token1 = insert(:oauth_token, user: admin, scopes: ["admin"]) + good_token2 = insert(:oauth_token, user: admin, scopes: ["admin:read"]) + good_token3 = insert(:oauth_token, user: admin, scopes: ["admin:read:accounts"]) + + bad_token1 = insert(:oauth_token, user: admin, scopes: ["read:accounts"]) + bad_token2 = insert(:oauth_token, user: admin, scopes: ["admin:read:accounts:partial"]) + bad_token3 = nil + + for good_token <- [good_token1, good_token2, good_token3] do + conn = + build_conn() + |> assign(:user, admin) + |> assign(:token, good_token) + |> get(url) + + assert json_response(conn, 200) + end + + for good_token <- [good_token1, good_token2, good_token3] do + conn = + build_conn() + |> assign(:user, nil) + |> assign(:token, good_token) + |> get(url) + + assert json_response(conn, :forbidden) + end + + for bad_token <- [bad_token1, bad_token2, bad_token3] do + conn = + build_conn() + |> assign(:user, admin) + |> assign(:token, bad_token) + |> get(url) + + assert json_response(conn, :forbidden) + end + end + end + + describe "unless [:auth, :enforce_oauth_admin_scope_usage]," do + setup do: clear_config([:auth, :enforce_oauth_admin_scope_usage], false) + + test "GET /api/pleroma/admin/users/:nickname requires " <> + "read:accounts or admin:read:accounts or broader scope", + %{admin: admin} do + user = insert(:user) + url = "/api/pleroma/admin/users/#{user.nickname}" + + good_token1 = insert(:oauth_token, user: admin, scopes: ["admin"]) + good_token2 = insert(:oauth_token, user: admin, scopes: ["admin:read"]) + good_token3 = insert(:oauth_token, user: admin, scopes: ["admin:read:accounts"]) + good_token4 = insert(:oauth_token, user: admin, scopes: ["read:accounts"]) + good_token5 = insert(:oauth_token, user: admin, scopes: ["read"]) + + good_tokens = [good_token1, good_token2, good_token3, good_token4, good_token5] + + bad_token1 = insert(:oauth_token, user: admin, scopes: ["read:accounts:partial"]) + bad_token2 = insert(:oauth_token, user: admin, scopes: ["admin:read:accounts:partial"]) + bad_token3 = nil + + for good_token <- good_tokens do + conn = + build_conn() + |> assign(:user, admin) + |> assign(:token, good_token) + |> get(url) + + assert json_response(conn, 200) + end + + for good_token <- good_tokens do + conn = + build_conn() + |> assign(:user, nil) + |> assign(:token, good_token) + |> get(url) + + assert json_response(conn, :forbidden) + end + + for bad_token <- [bad_token1, bad_token2, bad_token3] do + conn = + build_conn() + |> assign(:user, admin) + |> assign(:token, bad_token) + |> get(url) + + assert json_response(conn, :forbidden) + end + end + end + + describe "DELETE /api/pleroma/admin/users" do + test "single user", %{admin: admin, conn: conn} do + clear_config([:instance, :federating], true) + + user = + insert(:user, + avatar: %{"url" => [%{"href" => "https://someurl"}]}, + banner: %{"url" => [%{"href" => "https://somebanner"}]}, + bio: "Hello world!", + name: "A guy" + ) + + # Create some activities to check they got deleted later + follower = insert(:user) + {:ok, _} = CommonAPI.post(user, %{status: "test"}) + {:ok, _, _, _} = CommonAPI.follow(user, follower) + {:ok, _, _, _} = CommonAPI.follow(follower, user) + user = Repo.get(User, user.id) + assert user.note_count == 1 + assert user.follower_count == 1 + assert user.following_count == 1 + refute user.deactivated + + with_mock Pleroma.Web.Federator, + publish: fn _ -> nil end, + perform: fn _, _ -> nil end do + conn = + conn + |> put_req_header("accept", "application/json") + |> delete("/api/pleroma/admin/users?nickname=#{user.nickname}") + + ObanHelpers.perform_all() + + assert User.get_by_nickname(user.nickname).deactivated + + log_entry = Repo.one(ModerationLog) + + assert ModerationLog.get_log_entry_message(log_entry) == + "@#{admin.nickname} deleted users: @#{user.nickname}" + + assert json_response(conn, 200) == [user.nickname] + + user = Repo.get(User, user.id) + assert user.deactivated + + assert user.avatar == %{} + assert user.banner == %{} + assert user.note_count == 0 + assert user.follower_count == 0 + assert user.following_count == 0 + assert user.bio == "" + assert user.name == nil + + assert called(Pleroma.Web.Federator.publish(:_)) + end + end + + test "multiple users", %{admin: admin, conn: conn} do + user_one = insert(:user) + user_two = insert(:user) + + conn = + conn + |> put_req_header("accept", "application/json") + |> delete("/api/pleroma/admin/users", %{ + nicknames: [user_one.nickname, user_two.nickname] + }) + + log_entry = Repo.one(ModerationLog) + + assert ModerationLog.get_log_entry_message(log_entry) == + "@#{admin.nickname} deleted users: @#{user_one.nickname}, @#{user_two.nickname}" + + response = json_response(conn, 200) + assert response -- [user_one.nickname, user_two.nickname] == [] + end + end + + describe "/api/pleroma/admin/users" do + test "Create", %{conn: conn} do + conn = + conn + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/users", %{ + "users" => [ + %{ + "nickname" => "lain", + "email" => "lain@example.org", + "password" => "test" + }, + %{ + "nickname" => "lain2", + "email" => "lain2@example.org", + "password" => "test" + } + ] + }) + + response = json_response(conn, 200) |> Enum.map(&Map.get(&1, "type")) + assert response == ["success", "success"] + + log_entry = Repo.one(ModerationLog) + + assert ["lain", "lain2"] -- Enum.map(log_entry.data["subjects"], & &1["nickname"]) == [] + end + + test "Cannot create user with existing email", %{conn: conn} do + user = insert(:user) + + conn = + conn + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/users", %{ + "users" => [ + %{ + "nickname" => "lain", + "email" => user.email, + "password" => "test" + } + ] + }) + + assert json_response(conn, 409) == [ + %{ + "code" => 409, + "data" => %{ + "email" => user.email, + "nickname" => "lain" + }, + "error" => "email has already been taken", + "type" => "error" + } + ] + end + + test "Cannot create user with existing nickname", %{conn: conn} do + user = insert(:user) + + conn = + conn + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/users", %{ + "users" => [ + %{ + "nickname" => user.nickname, + "email" => "someuser@plerama.social", + "password" => "test" + } + ] + }) + + assert json_response(conn, 409) == [ + %{ + "code" => 409, + "data" => %{ + "email" => "someuser@plerama.social", + "nickname" => user.nickname + }, + "error" => "nickname has already been taken", + "type" => "error" + } + ] + end + + test "Multiple user creation works in transaction", %{conn: conn} do + user = insert(:user) + + conn = + conn + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/users", %{ + "users" => [ + %{ + "nickname" => "newuser", + "email" => "newuser@pleroma.social", + "password" => "test" + }, + %{ + "nickname" => "lain", + "email" => user.email, + "password" => "test" + } + ] + }) + + assert json_response(conn, 409) == [ + %{ + "code" => 409, + "data" => %{ + "email" => user.email, + "nickname" => "lain" + }, + "error" => "email has already been taken", + "type" => "error" + }, + %{ + "code" => 409, + "data" => %{ + "email" => "newuser@pleroma.social", + "nickname" => "newuser" + }, + "error" => "", + "type" => "error" + } + ] + + assert User.get_by_nickname("newuser") === nil + end + end + + describe "/api/pleroma/admin/users/:nickname" do + test "Show", %{conn: conn} do + user = insert(:user) + + conn = get(conn, "/api/pleroma/admin/users/#{user.nickname}") + + assert user_response(user) == json_response(conn, 200) + end + + test "when the user doesn't exist", %{conn: conn} do + user = build(:user) + + conn = get(conn, "/api/pleroma/admin/users/#{user.nickname}") + + assert %{"error" => "Not found"} == json_response(conn, 404) + end + end + + describe "/api/pleroma/admin/users/follow" do + test "allows to force-follow another user", %{admin: admin, conn: conn} do + user = insert(:user) + follower = insert(:user) + + conn + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/users/follow", %{ + "follower" => follower.nickname, + "followed" => user.nickname + }) + + user = User.get_cached_by_id(user.id) + follower = User.get_cached_by_id(follower.id) + + assert User.following?(follower, user) + + log_entry = Repo.one(ModerationLog) + + assert ModerationLog.get_log_entry_message(log_entry) == + "@#{admin.nickname} made @#{follower.nickname} follow @#{user.nickname}" + end + end + + describe "/api/pleroma/admin/users/unfollow" do + test "allows to force-unfollow another user", %{admin: admin, conn: conn} do + user = insert(:user) + follower = insert(:user) + + User.follow(follower, user) + + conn + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/users/unfollow", %{ + "follower" => follower.nickname, + "followed" => user.nickname + }) + + user = User.get_cached_by_id(user.id) + follower = User.get_cached_by_id(follower.id) + + refute User.following?(follower, user) + + log_entry = Repo.one(ModerationLog) + + assert ModerationLog.get_log_entry_message(log_entry) == + "@#{admin.nickname} made @#{follower.nickname} unfollow @#{user.nickname}" + end + end + + describe "GET /api/pleroma/admin/users" do + test "renders users array for the first page", %{conn: conn, admin: admin} do + user = insert(:user, local: false, tags: ["foo", "bar"]) + user2 = insert(:user, approval_pending: true, registration_reason: "I'm a chill dude") + + conn = get(conn, "/api/pleroma/admin/users?page=1") + + users = + [ + user_response( + admin, + %{"roles" => %{"admin" => true, "moderator" => false}} + ), + user_response(user, %{"local" => false, "tags" => ["foo", "bar"]}), + user_response( + user2, + %{ + "local" => true, + "approval_pending" => true, + "registration_reason" => "I'm a chill dude", + "actor_type" => "Person" + } + ) + ] + |> Enum.sort_by(& &1["nickname"]) + + assert json_response(conn, 200) == %{ + "count" => 3, + "page_size" => 50, + "users" => users + } + end + + test "pagination works correctly with service users", %{conn: conn} do + service1 = User.get_or_create_service_actor_by_ap_id(Web.base_url() <> "/meido", "meido") + + insert_list(25, :user) + + assert %{"count" => 26, "page_size" => 10, "users" => users1} = + conn + |> get("/api/pleroma/admin/users?page=1&filters=", %{page_size: "10"}) + |> json_response(200) + + assert Enum.count(users1) == 10 + assert service1 not in users1 + + assert %{"count" => 26, "page_size" => 10, "users" => users2} = + conn + |> get("/api/pleroma/admin/users?page=2&filters=", %{page_size: "10"}) + |> json_response(200) + + assert Enum.count(users2) == 10 + assert service1 not in users2 + + assert %{"count" => 26, "page_size" => 10, "users" => users3} = + conn + |> get("/api/pleroma/admin/users?page=3&filters=", %{page_size: "10"}) + |> json_response(200) + + assert Enum.count(users3) == 6 + assert service1 not in users3 + end + + test "renders empty array for the second page", %{conn: conn} do + insert(:user) + + conn = get(conn, "/api/pleroma/admin/users?page=2") + + assert json_response(conn, 200) == %{ + "count" => 2, + "page_size" => 50, + "users" => [] + } + end + + test "regular search", %{conn: conn} do + user = insert(:user, nickname: "bob") + + conn = get(conn, "/api/pleroma/admin/users?query=bo") + + assert json_response(conn, 200) == %{ + "count" => 1, + "page_size" => 50, + "users" => [user_response(user, %{"local" => true})] + } + end + + test "search by domain", %{conn: conn} do + user = insert(:user, nickname: "nickname@domain.com") + insert(:user) + + conn = get(conn, "/api/pleroma/admin/users?query=domain.com") + + assert json_response(conn, 200) == %{ + "count" => 1, + "page_size" => 50, + "users" => [user_response(user)] + } + end + + test "search by full nickname", %{conn: conn} do + user = insert(:user, nickname: "nickname@domain.com") + insert(:user) + + conn = get(conn, "/api/pleroma/admin/users?query=nickname@domain.com") + + assert json_response(conn, 200) == %{ + "count" => 1, + "page_size" => 50, + "users" => [user_response(user)] + } + end + + test "search by display name", %{conn: conn} do + user = insert(:user, name: "Display name") + insert(:user) + + conn = get(conn, "/api/pleroma/admin/users?name=display") + + assert json_response(conn, 200) == %{ + "count" => 1, + "page_size" => 50, + "users" => [user_response(user)] + } + end + + test "search by email", %{conn: conn} do + user = insert(:user, email: "email@example.com") + insert(:user) + + conn = get(conn, "/api/pleroma/admin/users?email=email@example.com") + + assert json_response(conn, 200) == %{ + "count" => 1, + "page_size" => 50, + "users" => [user_response(user)] + } + end + + test "regular search with page size", %{conn: conn} do + user = insert(:user, nickname: "aalice") + user2 = insert(:user, nickname: "alice") + + conn1 = get(conn, "/api/pleroma/admin/users?query=a&page_size=1&page=1") + + assert json_response(conn1, 200) == %{ + "count" => 2, + "page_size" => 1, + "users" => [user_response(user)] + } + + conn2 = get(conn, "/api/pleroma/admin/users?query=a&page_size=1&page=2") + + assert json_response(conn2, 200) == %{ + "count" => 2, + "page_size" => 1, + "users" => [user_response(user2)] + } + end + + test "only local users" do + admin = insert(:user, is_admin: true, nickname: "john") + token = insert(:oauth_admin_token, user: admin) + user = insert(:user, nickname: "bob") + + insert(:user, nickname: "bobb", local: false) + + conn = + build_conn() + |> assign(:user, admin) + |> assign(:token, token) + |> get("/api/pleroma/admin/users?query=bo&filters=local") + + assert json_response(conn, 200) == %{ + "count" => 1, + "page_size" => 50, + "users" => [user_response(user)] + } + end + + test "only local users with no query", %{conn: conn, admin: old_admin} do + admin = insert(:user, is_admin: true, nickname: "john") + user = insert(:user, nickname: "bob") + + insert(:user, nickname: "bobb", local: false) + + conn = get(conn, "/api/pleroma/admin/users?filters=local") + + users = + [ + user_response(user), + user_response(admin, %{ + "roles" => %{"admin" => true, "moderator" => false} + }), + user_response(old_admin, %{ + "deactivated" => false, + "roles" => %{"admin" => true, "moderator" => false} + }) + ] + |> Enum.sort_by(& &1["nickname"]) + + assert json_response(conn, 200) == %{ + "count" => 3, + "page_size" => 50, + "users" => users + } + end + + test "only unconfirmed users", %{conn: conn} do + sad_user = insert(:user, nickname: "sadboy", confirmation_pending: true) + old_user = insert(:user, nickname: "oldboy", confirmation_pending: true) + + insert(:user, nickname: "happyboy", approval_pending: false) + insert(:user, confirmation_pending: false) + + result = + conn + |> get("/api/pleroma/admin/users?filters=need_confirmed") + |> json_response(200) + + users = + Enum.map([old_user, sad_user], fn user -> + user_response(user, %{ + "confirmation_pending" => true, + "approval_pending" => false + }) + end) + |> Enum.sort_by(& &1["nickname"]) + + assert result == %{"count" => 2, "page_size" => 50, "users" => users} + end + + test "only unapproved users", %{conn: conn} do + user = + insert(:user, + nickname: "sadboy", + approval_pending: true, + registration_reason: "Plz let me in!" + ) + + insert(:user, nickname: "happyboy", approval_pending: false) + + conn = get(conn, "/api/pleroma/admin/users?filters=need_approval") + + users = + [ + %{ + "deactivated" => user.deactivated, + "id" => user.id, + "nickname" => user.nickname, + "roles" => %{"admin" => false, "moderator" => false}, + "local" => true, + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname), + "confirmation_pending" => false, + "approval_pending" => true, + "url" => user.ap_id, + "registration_reason" => "Plz let me in!", + "actor_type" => "Person" + } + ] + |> Enum.sort_by(& &1["nickname"]) + + assert json_response(conn, 200) == %{ + "count" => 1, + "page_size" => 50, + "users" => users + } + end + + test "load only admins", %{conn: conn, admin: admin} do + second_admin = insert(:user, is_admin: true) + insert(:user) + insert(:user) + + conn = get(conn, "/api/pleroma/admin/users?filters=is_admin") + + users = + [ + %{ + "deactivated" => false, + "id" => admin.id, + "nickname" => admin.nickname, + "roles" => %{"admin" => true, "moderator" => false}, + "local" => admin.local, + "tags" => [], + "avatar" => User.avatar_url(admin) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(admin.name || admin.nickname), + "confirmation_pending" => false, + "approval_pending" => false, + "url" => admin.ap_id, + "registration_reason" => nil, + "actor_type" => "Person" + }, + %{ + "deactivated" => false, + "id" => second_admin.id, + "nickname" => second_admin.nickname, + "roles" => %{"admin" => true, "moderator" => false}, + "local" => second_admin.local, + "tags" => [], + "avatar" => User.avatar_url(second_admin) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(second_admin.name || second_admin.nickname), + "confirmation_pending" => false, + "approval_pending" => false, + "url" => second_admin.ap_id, + "registration_reason" => nil, + "actor_type" => "Person" + } + ] + |> Enum.sort_by(& &1["nickname"]) + + assert json_response(conn, 200) == %{ + "count" => 2, + "page_size" => 50, + "users" => users + } + end + + test "load only moderators", %{conn: conn} do + moderator = insert(:user, is_moderator: true) + insert(:user) + insert(:user) + + conn = get(conn, "/api/pleroma/admin/users?filters=is_moderator") + + assert json_response(conn, 200) == %{ + "count" => 1, + "page_size" => 50, + "users" => [ + %{ + "deactivated" => false, + "id" => moderator.id, + "nickname" => moderator.nickname, + "roles" => %{"admin" => false, "moderator" => true}, + "local" => moderator.local, + "tags" => [], + "avatar" => User.avatar_url(moderator) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(moderator.name || moderator.nickname), + "confirmation_pending" => false, + "approval_pending" => false, + "url" => moderator.ap_id, + "registration_reason" => nil, + "actor_type" => "Person" + } + ] + } + end + + test "load users with tags list", %{conn: conn} do + user1 = insert(:user, tags: ["first"]) + user2 = insert(:user, tags: ["second"]) + insert(:user) + insert(:user) + + conn = get(conn, "/api/pleroma/admin/users?tags[]=first&tags[]=second") + + users = + [ + user_response( + user1, + %{ + "deactivated" => false, + "roles" => %{"admin" => false, "moderator" => false}, + "local" => user1.local, + "tags" => ["first"], + "avatar" => User.avatar_url(user1) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user1.name || user1.nickname), + "confirmation_pending" => false, + "approval_pending" => false, + "url" => user1.ap_id, + "registration_reason" => nil, + "actor_type" => "Person" + } + ), + %{ + "deactivated" => false, + "id" => user2.id, + "nickname" => user2.nickname, + "roles" => %{"admin" => false, "moderator" => false}, + "local" => user2.local, + "tags" => ["second"], + "avatar" => User.avatar_url(user2) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user2.name || user2.nickname), + "confirmation_pending" => false, + "approval_pending" => false, + "url" => user2.ap_id, + "registration_reason" => nil, + "actor_type" => "Person" + } + ] + |> Enum.sort_by(& &1["nickname"]) + + assert json_response(conn, 200) == %{ + "count" => 2, + "page_size" => 50, + "users" => users + } + end + + test "`active` filters out users pending approval", %{token: token} do + insert(:user, approval_pending: true) + %{id: user_id} = insert(:user, approval_pending: false) + %{id: admin_id} = token.user + + conn = + build_conn() + |> assign(:user, token.user) + |> assign(:token, token) + |> get("/api/pleroma/admin/users?filters=active") + + assert %{ + "count" => 2, + "page_size" => 50, + "users" => [ + %{"id" => ^admin_id}, + %{"id" => ^user_id} + ] + } = json_response(conn, 200) + end + + test "it works with multiple filters" do + admin = insert(:user, nickname: "john", is_admin: true) + token = insert(:oauth_admin_token, user: admin) + user = insert(:user, nickname: "bob", local: false, deactivated: true) + + insert(:user, nickname: "ken", local: true, deactivated: true) + insert(:user, nickname: "bobb", local: false, deactivated: false) + + conn = + build_conn() + |> assign(:user, admin) + |> assign(:token, token) + |> get("/api/pleroma/admin/users?filters=deactivated,external") + + assert json_response(conn, 200) == %{ + "count" => 1, + "page_size" => 50, + "users" => [user_response(user)] + } + end + + test "it omits relay user", %{admin: admin, conn: conn} do + assert %User{} = Relay.get_actor() + + conn = get(conn, "/api/pleroma/admin/users") + + assert json_response(conn, 200) == %{ + "count" => 1, + "page_size" => 50, + "users" => [ + user_response(admin, %{"roles" => %{"admin" => true, "moderator" => false}}) + ] + } + end + end + + test "PATCH /api/pleroma/admin/users/activate", %{admin: admin, conn: conn} do + user_one = insert(:user, deactivated: true) + user_two = insert(:user, deactivated: true) + + conn = + patch( + conn, + "/api/pleroma/admin/users/activate", + %{nicknames: [user_one.nickname, user_two.nickname]} + ) + + response = json_response(conn, 200) + assert Enum.map(response["users"], & &1["deactivated"]) == [false, false] + + log_entry = Repo.one(ModerationLog) + + assert ModerationLog.get_log_entry_message(log_entry) == + "@#{admin.nickname} activated users: @#{user_one.nickname}, @#{user_two.nickname}" + end + + test "PATCH /api/pleroma/admin/users/deactivate", %{admin: admin, conn: conn} do + user_one = insert(:user, deactivated: false) + user_two = insert(:user, deactivated: false) + + conn = + patch( + conn, + "/api/pleroma/admin/users/deactivate", + %{nicknames: [user_one.nickname, user_two.nickname]} + ) + + response = json_response(conn, 200) + assert Enum.map(response["users"], & &1["deactivated"]) == [true, true] + + log_entry = Repo.one(ModerationLog) + + assert ModerationLog.get_log_entry_message(log_entry) == + "@#{admin.nickname} deactivated users: @#{user_one.nickname}, @#{user_two.nickname}" + end + + test "PATCH /api/pleroma/admin/users/approve", %{admin: admin, conn: conn} do + user_one = insert(:user, approval_pending: true) + user_two = insert(:user, approval_pending: true) + + conn = + patch( + conn, + "/api/pleroma/admin/users/approve", + %{nicknames: [user_one.nickname, user_two.nickname]} + ) + + response = json_response(conn, 200) + assert Enum.map(response["users"], & &1["approval_pending"]) == [false, false] + + log_entry = Repo.one(ModerationLog) + + assert ModerationLog.get_log_entry_message(log_entry) == + "@#{admin.nickname} approved users: @#{user_one.nickname}, @#{user_two.nickname}" + end + + test "PATCH /api/pleroma/admin/users/:nickname/toggle_activation", %{admin: admin, conn: conn} do + user = insert(:user) + + conn = patch(conn, "/api/pleroma/admin/users/#{user.nickname}/toggle_activation") + + assert json_response(conn, 200) == + user_response( + user, + %{"deactivated" => !user.deactivated} + ) + + log_entry = Repo.one(ModerationLog) + + assert ModerationLog.get_log_entry_message(log_entry) == + "@#{admin.nickname} deactivated users: @#{user.nickname}" + end + + defp user_response(user, attrs \\ %{}) do + %{ + "deactivated" => user.deactivated, + "id" => user.id, + "nickname" => user.nickname, + "roles" => %{"admin" => false, "moderator" => false}, + "local" => user.local, + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname), + "confirmation_pending" => false, + "approval_pending" => false, + "url" => user.ap_id, + "registration_reason" => nil, + "actor_type" => "Person" + } + |> Map.merge(attrs) + end +end From 46b420aa602050d7b3bff33a6b51d54852b2adb3 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Fri, 25 Sep 2020 09:39:49 +0300 Subject: [PATCH 57/75] need_confirmed -> unconfirmed --- lib/pleroma/user/query.ex | 4 ++-- lib/pleroma/web/admin_api/controllers/user_controller.ex | 2 +- test/pleroma/web/admin_api/search_test.exs | 2 +- test/web/admin_api/controllers/user_controller_test.exs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex index 2933e7fb4..711439eef 100644 --- a/lib/pleroma/user/query.ex +++ b/lib/pleroma/user/query.ex @@ -43,7 +43,7 @@ defmodule Pleroma.User.Query do active: boolean(), deactivated: boolean(), need_approval: boolean(), - need_confirmed: boolean(), + unconfirmed: boolean(), is_admin: boolean(), is_moderator: boolean(), super_users: boolean(), @@ -157,7 +157,7 @@ defp compose_query({:need_approval, _}, query) do where(query, [u], u.approval_pending) end - defp compose_query({:need_confirmed, _}, query) do + defp compose_query({:unconfirmed, _}, query) do where(query, [u], u.confirmation_pending) end diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex index 5e049aa0f..db6ef3220 100644 --- a/lib/pleroma/web/admin_api/controllers/user_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex @@ -259,7 +259,7 @@ def list(conn, params) do end end - @filters ~w(local external active deactivated need_approval need_confirmed is_admin is_moderator) + @filters ~w(local external active deactivated need_approval unconfirmed is_admin is_moderator) @spec maybe_parse_filters(String.t()) :: %{required(String.t()) => true} | %{} defp maybe_parse_filters(filters) when is_nil(filters) or filters == "", do: %{} diff --git a/test/pleroma/web/admin_api/search_test.exs b/test/pleroma/web/admin_api/search_test.exs index 27ca396e6..82da86f7f 100644 --- a/test/pleroma/web/admin_api/search_test.exs +++ b/test/pleroma/web/admin_api/search_test.exs @@ -184,7 +184,7 @@ test "it returns unconfirmed user" do insert(:user) {:ok, _results, total} = Search.user() - {:ok, [^unconfirmed], count} = Search.user(%{need_confirmed: true}) + {:ok, [^unconfirmed], count} = Search.user(%{unconfirmed: true}) assert total == 3 assert count == 1 end diff --git a/test/web/admin_api/controllers/user_controller_test.exs b/test/web/admin_api/controllers/user_controller_test.exs index 347258058..6384f8ddc 100644 --- a/test/web/admin_api/controllers/user_controller_test.exs +++ b/test/web/admin_api/controllers/user_controller_test.exs @@ -644,7 +644,7 @@ test "only unconfirmed users", %{conn: conn} do result = conn - |> get("/api/pleroma/admin/users?filters=need_confirmed") + |> get("/api/pleroma/admin/users?filters=unconfirmed") |> json_response(200) users = From 60663150b5b936c831bdc0cfeade30867e536317 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Fri, 25 Sep 2020 16:04:01 +0300 Subject: [PATCH 58/75] admin user search: added filter by `actor_type` --- lib/pleroma/user/query.ex | 7 ++++++- test/pleroma/web/admin_api/search_test.exs | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex index 711439eef..7ef2a1455 100644 --- a/lib/pleroma/user/query.ex +++ b/lib/pleroma/user/query.ex @@ -56,7 +56,8 @@ defmodule Pleroma.User.Query do ap_id: [String.t()], order_by: term(), select: term(), - limit: pos_integer() + limit: pos_integer(), + actor_types: [String.t()] } | map() @@ -115,6 +116,10 @@ defp compose_query({:is_admin, bool}, query) do where(query, [u], u.is_admin == ^bool) end + defp compose_query({:actor_types, actor_types}, query) when is_list(actor_types) do + where(query, [u], u.actor_type in ^actor_types) + end + defp compose_query({:is_moderator, bool}, query) do where(query, [u], u.is_moderator == ^bool) end diff --git a/test/pleroma/web/admin_api/search_test.exs b/test/pleroma/web/admin_api/search_test.exs index 82da86f7f..92a116c65 100644 --- a/test/pleroma/web/admin_api/search_test.exs +++ b/test/pleroma/web/admin_api/search_test.exs @@ -143,6 +143,20 @@ test "it returns users with tags" do assert user2 in users end + test "it returns users by actor_types" do + user_service = insert(:user, actor_type: "Service") + user_application = insert(:user, actor_type: "Application") + user1 = insert(:user) + user2 = insert(:user) + + {:ok, [^user_service], 1} = Search.user(%{actor_types: ["Service"]}) + {:ok, [^user_application], 1} = Search.user(%{actor_types: ["Application"]}) + {:ok, [^user1, ^user2], 2} = Search.user(%{actor_types: ["Person"]}) + + {:ok, [^user_service, ^user1, ^user2], 3} = + Search.user(%{actor_types: ["Person", "Service"]}) + end + test "it returns user by display name" do user = insert(:user, name: "Display name") insert(:user) From 44e5a57d1a1b36ad2971763f3a8cab82bdb94b08 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Fri, 25 Sep 2020 16:37:55 +0300 Subject: [PATCH 59/75] admin api: added user filters by `actor_types` --- .../admin_api/controllers/user_controller.ex | 3 +- .../controllers/user_controller_test.exs | 164 ++++++++---------- 2 files changed, 76 insertions(+), 91 deletions(-) diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex index db6ef3220..15cec9b09 100644 --- a/lib/pleroma/web/admin_api/controllers/user_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex @@ -243,7 +243,8 @@ def list(conn, params) do page_size: page_size, tags: params["tags"], name: params["name"], - email: params["email"] + email: params["email"], + actor_types: params["actor_types"] } |> Map.merge(filters) diff --git a/test/web/admin_api/controllers/user_controller_test.exs b/test/web/admin_api/controllers/user_controller_test.exs index 6384f8ddc..b8a927e56 100644 --- a/test/web/admin_api/controllers/user_controller_test.exs +++ b/test/web/admin_api/controllers/user_controller_test.exs @@ -673,23 +673,10 @@ test "only unapproved users", %{conn: conn} do users = [ - %{ - "deactivated" => user.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => true, - "tags" => [], - "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname), - "confirmation_pending" => false, - "approval_pending" => true, - "url" => user.ap_id, - "registration_reason" => "Plz let me in!", - "actor_type" => "Person" - } + user_response(user, + %{"approval_pending" => true, "registration_reason" => "Plz let me in!"} + ) ] - |> Enum.sort_by(& &1["nickname"]) assert json_response(conn, 200) == %{ "count" => 1, @@ -707,36 +694,14 @@ test "load only admins", %{conn: conn, admin: admin} do users = [ - %{ + user_response(admin, %{ "deactivated" => false, - "id" => admin.id, - "nickname" => admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "local" => admin.local, - "tags" => [], - "avatar" => User.avatar_url(admin) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(admin.name || admin.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => admin.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - }, - %{ + "roles" => %{"admin" => true, "moderator" => false} + }), + user_response(second_admin, %{ "deactivated" => false, - "id" => second_admin.id, - "nickname" => second_admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "local" => second_admin.local, - "tags" => [], - "avatar" => User.avatar_url(second_admin) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(second_admin.name || second_admin.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => second_admin.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } + "roles" => %{"admin" => true, "moderator" => false} + }) ] |> Enum.sort_by(& &1["nickname"]) @@ -758,25 +723,73 @@ test "load only moderators", %{conn: conn} do "count" => 1, "page_size" => 50, "users" => [ - %{ - "deactivated" => false, - "id" => moderator.id, - "nickname" => moderator.nickname, - "roles" => %{"admin" => false, "moderator" => true}, - "local" => moderator.local, - "tags" => [], - "avatar" => User.avatar_url(moderator) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(moderator.name || moderator.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => moderator.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } + user_response(moderator, %{ + "deactivated" => false, + "roles" => %{"admin" => false, "moderator" => true} + }) ] } end + test "load users with actor_type is Person", %{admin: admin, conn: conn} do + insert(:user, actor_type: "Service") + insert(:user, actor_type: "Application") + + user1 = insert(:user) + user2 = insert(:user) + + response = conn + |> get(user_path(conn, :list), %{actor_types: ["Person"]}) + |> json_response(200) + + users = + [ + user_response(admin, %{"roles" => %{"admin" => true, "moderator" => false}}), + user_response(user1), + user_response(user2) + ] + |> Enum.sort_by(& &1["nickname"]) + + assert response == %{"count" => 3, "page_size" => 50, "users" => users} + end + + test "load users with actor_type is Person and Service", %{admin: admin, conn: conn} do + user_service = insert(:user, actor_type: "Service") + insert(:user, actor_type: "Application") + + user1 = insert(:user) + user2 = insert(:user) + + response = conn + |> get(user_path(conn, :list), %{actor_types: ["Person", "Service"]}) + |> json_response(200) + + users = + [ + user_response(admin, %{"roles" => %{"admin" => true, "moderator" => false}}), + user_response(user1), + user_response(user2), + user_response(user_service, %{"actor_type" => "Service"}) + ] + |> Enum.sort_by(& &1["nickname"]) + + assert response == %{"count" => 4, "page_size" => 50, "users" => users} + end + + test "load users with actor_type is Service", %{conn: conn} do + user_service = insert(:user, actor_type: "Service") + insert(:user, actor_type: "Application") + insert(:user) + insert(:user) + + response = conn + |> get(user_path(conn, :list), %{actor_types: ["Service"]}) + |> json_response(200) + users = [user_response(user_service, %{"actor_type" => "Service"})] + + assert response == %{"count" => 1, "page_size" => 50, "users" => users } + end + test "load users with tags list", %{conn: conn} do user1 = insert(:user, tags: ["first"]) user2 = insert(:user, tags: ["second"]) @@ -787,37 +800,8 @@ test "load users with tags list", %{conn: conn} do users = [ - user_response( - user1, - %{ - "deactivated" => false, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => user1.local, - "tags" => ["first"], - "avatar" => User.avatar_url(user1) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user1.name || user1.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => user1.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } - ), - %{ - "deactivated" => false, - "id" => user2.id, - "nickname" => user2.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => user2.local, - "tags" => ["second"], - "avatar" => User.avatar_url(user2) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user2.name || user2.nickname), - "confirmation_pending" => false, - "approval_pending" => false, - "url" => user2.ap_id, - "registration_reason" => nil, - "actor_type" => "Person" - } + user_response(user1, %{"tags" => ["first"]}), + user_response(user2, %{"tags" => ["second"]}) ] |> Enum.sort_by(& &1["nickname"]) From 51189ad365a462163a4c98a7b5cb7ae74f2d634a Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Fri, 25 Sep 2020 16:40:50 +0300 Subject: [PATCH 60/75] update docs --- docs/API/admin_api.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index 7bf13daef..f7b5bcae7 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -20,12 +20,14 @@ Configuration options: - `external`: only external users - `active`: only active users - `need_approval`: only unapproved users + - `unconfirmed`: only unconfirmed users - `deactivated`: only deactivated users - `is_admin`: users with admin role - `is_moderator`: users with moderator role - *optional* `page`: **integer** page number - *optional* `page_size`: **integer** number of users per page (default is `50`) - *optional* `tags`: **[string]** tags list + - *optional* `actor_types`: **[string]** actor type list (`Person`, `Service`, `Application`) - *optional* `name`: **string** user display name - *optional* `email`: **string** user email - Example: `https://mypleroma.org/api/pleroma/admin/users?query=john&filters=local,active&page=1&page_size=10&tags[]=some_tag&tags[]=another_tag&name=display_name&email=email@example.com` From add26817e3a8756d2f6aea256c6d4b2070a8070b Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Fri, 25 Sep 2020 16:50:47 +0300 Subject: [PATCH 61/75] update changelog --- CHANGELOG.md | 2 + .../controllers/user_controller_test.exs | 42 ++++++++++--------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05e94581a..82c5eaf8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Pleroma API: Importing the mutes users from CSV files. - Admin API: Importing emoji from a zip file - Pleroma API: Pagination for remote/local packs and emoji. +- Admin API: (`GET /api/pleroma/admin/users`) added filters user by `unconfirmed` status +- Admin API: (`GET /api/pleroma/admin/users`) added filters user by `actor_type` diff --git a/test/web/admin_api/controllers/user_controller_test.exs b/test/web/admin_api/controllers/user_controller_test.exs index b8a927e56..da26caf25 100644 --- a/test/web/admin_api/controllers/user_controller_test.exs +++ b/test/web/admin_api/controllers/user_controller_test.exs @@ -671,12 +671,12 @@ test "only unapproved users", %{conn: conn} do conn = get(conn, "/api/pleroma/admin/users?filters=need_approval") - users = - [ - user_response(user, - %{"approval_pending" => true, "registration_reason" => "Plz let me in!"} - ) - ] + users = [ + user_response( + user, + %{"approval_pending" => true, "registration_reason" => "Plz let me in!"} + ) + ] assert json_response(conn, 200) == %{ "count" => 1, @@ -724,9 +724,9 @@ test "load only moderators", %{conn: conn} do "page_size" => 50, "users" => [ user_response(moderator, %{ - "deactivated" => false, - "roles" => %{"admin" => false, "moderator" => true} - }) + "deactivated" => false, + "roles" => %{"admin" => false, "moderator" => true} + }) ] } end @@ -738,9 +738,10 @@ test "load users with actor_type is Person", %{admin: admin, conn: conn} do user1 = insert(:user) user2 = insert(:user) - response = conn - |> get(user_path(conn, :list), %{actor_types: ["Person"]}) - |> json_response(200) + response = + conn + |> get(user_path(conn, :list), %{actor_types: ["Person"]}) + |> json_response(200) users = [ @@ -760,9 +761,10 @@ test "load users with actor_type is Person and Service", %{admin: admin, conn: c user1 = insert(:user) user2 = insert(:user) - response = conn - |> get(user_path(conn, :list), %{actor_types: ["Person", "Service"]}) - |> json_response(200) + response = + conn + |> get(user_path(conn, :list), %{actor_types: ["Person", "Service"]}) + |> json_response(200) users = [ @@ -782,12 +784,14 @@ test "load users with actor_type is Service", %{conn: conn} do insert(:user) insert(:user) - response = conn - |> get(user_path(conn, :list), %{actor_types: ["Service"]}) - |> json_response(200) + response = + conn + |> get(user_path(conn, :list), %{actor_types: ["Service"]}) + |> json_response(200) + users = [user_response(user_service, %{"actor_type" => "Service"})] - assert response == %{"count" => 1, "page_size" => 50, "users" => users } + assert response == %{"count" => 1, "page_size" => 50, "users" => users} end test "load users with tags list", %{conn: conn} do From ef627b9391e0dde8adf03c0132fbc2eeac6bdede Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Thu, 22 Oct 2020 12:04:23 +0300 Subject: [PATCH 62/75] fix module name --- lib/pleroma/web/admin_api/controllers/user_controller.ex | 2 +- .../web/admin_api/controllers/user_controller_test.exs | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename test/{ => pleroma}/web/admin_api/controllers/user_controller_test.exs (100%) diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex index 15cec9b09..a2a1c875d 100644 --- a/lib/pleroma/web/admin_api/controllers/user_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex @@ -9,13 +9,13 @@ defmodule Pleroma.Web.AdminAPI.UserController do only: [fetch_integer_param: 3] alias Pleroma.ModerationLog - alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.User alias Pleroma.Web.ActivityPub.Builder alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.AdminAPI alias Pleroma.Web.AdminAPI.AccountView alias Pleroma.Web.AdminAPI.Search + alias Pleroma.Web.Plugs.OAuthScopesPlug @users_page_size 50 diff --git a/test/web/admin_api/controllers/user_controller_test.exs b/test/pleroma/web/admin_api/controllers/user_controller_test.exs similarity index 100% rename from test/web/admin_api/controllers/user_controller_test.exs rename to test/pleroma/web/admin_api/controllers/user_controller_test.exs From 60e379ce0b74bbe1b0f40a954aec040beab20e4e Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 23 Oct 2020 13:53:01 +0200 Subject: [PATCH 63/75] User: Correctly handle whitespace names. --- lib/pleroma/user.ex | 5 +- lib/pleroma/web/activity_pub/activity_pub.ex | 4 -- test/fixtures/mewmew_no_name.json | 46 +++++++++++++++++++ .../web/activity_pub/activity_pub_test.exs | 11 +++++ 4 files changed, 60 insertions(+), 6 deletions(-) create mode 100644 test/fixtures/mewmew_no_name.json diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index dc41d0001..72f507f1e 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -426,7 +426,6 @@ def remote_user_changeset(struct \\ %User{local: false}, params) do params, [ :bio, - :name, :emoji, :ap_id, :inbox, @@ -455,7 +454,9 @@ def remote_user_changeset(struct \\ %User{local: false}, params) do :accepts_chat_messages ] ) - |> validate_required([:name, :ap_id]) + |> cast(params, [:name], empty_values: []) + |> validate_required([:ap_id]) + |> validate_required([:name], trim: false) |> unique_constraint(:nickname) |> validate_format(:nickname, @email_regex) |> validate_length(:bio, max: bio_limit) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index d17c892a7..df18db603 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1378,10 +1378,6 @@ def fetch_and_prepare_user_from_ap_id(ap_id, opts \\ []) do {:ok, data} <- user_data_from_user_object(data) do {:ok, maybe_update_follow_information(data)} else - {:error, "Object has been deleted" = e} -> - Logger.debug("Could not decode user at fetch #{ap_id}, #{inspect(e)}") - {:error, e} - {:error, {:reject, reason} = e} -> Logger.info("Rejected user #{ap_id}: #{inspect(reason)}") {:error, e} diff --git a/test/fixtures/mewmew_no_name.json b/test/fixtures/mewmew_no_name.json new file mode 100644 index 000000000..532d4cf70 --- /dev/null +++ b/test/fixtures/mewmew_no_name.json @@ -0,0 +1,46 @@ +{ + "@context" : [ + "https://www.w3.org/ns/activitystreams", + "https://princess.cat/schemas/litepub-0.1.jsonld", + { + "@language" : "und" + } + ], + "attachment" : [], + "capabilities" : { + "acceptsChatMessages" : true + }, + "discoverable" : false, + "endpoints" : { + "oauthAuthorizationEndpoint" : "https://princess.cat/oauth/authorize", + "oauthRegistrationEndpoint" : "https://princess.cat/api/v1/apps", + "oauthTokenEndpoint" : "https://princess.cat/oauth/token", + "sharedInbox" : "https://princess.cat/inbox", + "uploadMedia" : "https://princess.cat/api/ap/upload_media" + }, + "followers" : "https://princess.cat/users/mewmew/followers", + "following" : "https://princess.cat/users/mewmew/following", + "icon" : { + "type" : "Image", + "url" : "https://princess.cat/media/12794fb50e86911e65be97f69196814049dcb398a2f8b58b99bb6591576e648c.png?name=blobcatpresentpink.png" + }, + "id" : "https://princess.cat/users/mewmew", + "image" : { + "type" : "Image", + "url" : "https://princess.cat/media/05d8bf3953ab6028fc920494ffc643fbee9dcef40d7bdd06f107e19acbfbd7f9.png" + }, + "inbox" : "https://princess.cat/users/mewmew/inbox", + "manuallyApprovesFollowers" : true, + "name" : " ", + "outbox" : "https://princess.cat/users/mewmew/outbox", + "preferredUsername" : "mewmew", + "publicKey" : { + "id" : "https://princess.cat/users/mewmew#main-key", + "owner" : "https://princess.cat/users/mewmew", + "publicKeyPem" : "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAru7VpygVef4zrFwnj0Mh\nrbO/2z2EdKN3rERtNrT8zWsLXNLQ50lfpRPnGDrd+xq7Rva4EIu0d5KJJ9n4vtY0\nuxK3On9vA2oyjLlR9O0lI3XTrHJborG3P7IPXrmNUMFpHiFHNqHp5tugUrs1gUFq\n7tmOmM92IP4Wjk8qNHFcsfnUbaPTX7sNIhteQKdi5HrTb/6lrEIe4G/FlMKRqxo3\nRNHuv6SNFQuiUKvFzjzazvjkjvBSm+aFROgdHa2tKl88StpLr7xmuY8qNFCRT6W0\nLacRp6c8ah5f03Kd+xCBVhCKvKaF1K0ERnQTBiitUh85md+Mtx/CoDoLnmpnngR3\nvQIDAQAB\n-----END PUBLIC KEY-----\n\n" + }, + "summary" : "please reply to my posts as direct messages if you have many followers", + "tag" : [], + "type" : "Person", + "url" : "https://princess.cat/users/mewmew" +} diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index 6ac883b23..43bd14ee6 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -2273,4 +2273,15 @@ test "`following` still contains self-replies by friends" do assert length(activities) == 2 end end + + test "allow fetching of accounts with an empty string name field" do + Tesla.Mock.mock(fn + %{method: :get, url: "https://princess.cat/users/mewmew"} -> + file = File.read!("test/fixtures/mewmew_no_name.json") + %Tesla.Env{status: 200, body: file} + end) + + {:ok, user} = ActivityPub.make_user_from_ap_id("https://princess.cat/users/mewmew") + assert user.name == " " + end end From a999e8b0b73372ed129c5a26acfe73e5f7478c5b Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 23 Oct 2020 13:55:08 +0200 Subject: [PATCH 64/75] Changelog: Add info about whitespace name remote users. --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index afeaa930b..1f15a8b95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,7 +47,8 @@ switched to a new configuration mechanism, however it was not officially removed - Add documented-but-missing chat pagination. - Allow sending out emails again. -- Allow sending chat messages to yourself +- Allow sending chat messages to yourself. +- Fix remote users with a whitespace name. ## Unreleased (Patch) From de6d49c8cec84a530f2835313c95064ae8df3604 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 26 Oct 2020 16:33:26 +0100 Subject: [PATCH 65/75] ActivityPub: Add back debug call + explanation. --- lib/pleroma/web/activity_pub/activity_pub.ex | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index df18db603..13869f897 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1378,6 +1378,11 @@ def fetch_and_prepare_user_from_ap_id(ap_id, opts \\ []) do {:ok, data} <- user_data_from_user_object(data) do {:ok, maybe_update_follow_information(data)} else + # If this has been deleted, only log a debug and not an error + {:error, "Object has been deleted" = e} -> + Logger.debug("Could not decode user at fetch #{ap_id}, #{inspect(e)}") + {:error, e} + {:error, {:reject, reason} = e} -> Logger.info("Rejected user #{ap_id}: #{inspect(reason)}") {:error, e} From cbe41408e4f77da55d59ee3d4d26d002a1f20f02 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 27 Oct 2020 14:37:48 -0500 Subject: [PATCH 66/75] phoenix_controller_render_duration is no longer available in telemetry of Phoenix 1.5+ --- test/pleroma/web/endpoint/metrics_exporter_test.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/pleroma/web/endpoint/metrics_exporter_test.exs b/test/pleroma/web/endpoint/metrics_exporter_test.exs index f954cc1e7..875addc96 100644 --- a/test/pleroma/web/endpoint/metrics_exporter_test.exs +++ b/test/pleroma/web/endpoint/metrics_exporter_test.exs @@ -38,7 +38,6 @@ test "serves app metrics", %{conn: conn} do for metric <- [ "http_requests_total", "http_request_duration_microseconds", - "phoenix_controller_render_duration", "phoenix_controller_call_duration", "telemetry_scrape_duration", "erlang_vm_memory_atom_bytes_total" From d28f72a55af9442719ff01fe7052802c285f6ea8 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 27 Oct 2020 22:58:55 +0300 Subject: [PATCH 67/75] FrontStatic plug: excluded invalid url --- lib/pleroma/web/plugs/frontend_static.ex | 26 +++++++++++-------- .../web/plugs/frontend_static_plug_test.exs | 21 +++++++++++++++ 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/web/plugs/frontend_static.ex b/lib/pleroma/web/plugs/frontend_static.ex index ceb10dcf8..1b0b36813 100644 --- a/lib/pleroma/web/plugs/frontend_static.ex +++ b/lib/pleroma/web/plugs/frontend_static.ex @@ -34,22 +34,26 @@ def init(opts) do end def call(conn, opts) do - frontend_type = Map.get(opts, :frontend_type, :primary) - path = file_path("", frontend_type) - - if path do - conn - |> call_static(opts, path) + with false <- invalid_path?(conn.path_info), + frontend_type <- Map.get(opts, :frontend_type, :primary), + path when not is_nil(path) <- file_path("", frontend_type) do + call_static(conn, opts, path) else - conn + _ -> + conn end end - defp call_static(conn, opts, from) do - opts = - opts - |> Map.put(:from, from) + defp invalid_path?(list) do + invalid_path?(list, :binary.compile_pattern(["/", "\\", ":", "\0"])) + end + defp invalid_path?([h | _], _match) when h in [".", "..", ""], do: true + defp invalid_path?([h | t], match), do: String.contains?(h, match) or invalid_path?(t) + defp invalid_path?([], _match), do: false + + defp call_static(conn, opts, from) do + opts = Map.put(opts, :from, from) Plug.Static.call(conn, opts) end end diff --git a/test/pleroma/web/plugs/frontend_static_plug_test.exs b/test/pleroma/web/plugs/frontend_static_plug_test.exs index f6f7d7bdb..8b7b022fc 100644 --- a/test/pleroma/web/plugs/frontend_static_plug_test.exs +++ b/test/pleroma/web/plugs/frontend_static_plug_test.exs @@ -4,6 +4,7 @@ defmodule Pleroma.Web.Plugs.FrontendStaticPlugTest do use Pleroma.Web.ConnCase + import Mock @dir "test/tmp/instance_static" @@ -53,4 +54,24 @@ test "overrides existing static files for the `pleroma/admin` path", %{conn: con index = get(conn, "/pleroma/admin/") assert html_response(index, 200) == "from frontend plug" end + + test "exclude invalid path", %{conn: conn} do + name = "pleroma-fe" + ref = "dist" + clear_config([:media_proxy, :enabled], true) + clear_config([Pleroma.Web.Endpoint, :secret_key_base], "00000000000") + clear_config([:frontends, :primary], %{"name" => name, "ref" => ref}) + path = "#{@dir}/frontends/#{name}/#{ref}" + + File.mkdir_p!("#{path}/proxy/rr/ss") + File.write!("#{path}/proxy/rr/ss/Ek7w8WPVcAApOvN.jpg:large", "FB image") + + url = + Pleroma.Web.MediaProxy.encode_url("https://pbs.twimg.com/media/Ek7w8WPVcAApOvN.jpg:large") + + with_mock Pleroma.ReverseProxy, + call: fn _conn, _url, _opts -> %Plug.Conn{status: :success} end do + assert %Plug.Conn{status: :success} = get(conn, url) + end + end end From 4f90077767b416f3469fe7c8acfaa6932c579ec2 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 28 Oct 2020 15:32:44 +0400 Subject: [PATCH 68/75] Fix warning --- test/pleroma/user/backup_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pleroma/user/backup_test.exs b/test/pleroma/user/backup_test.exs index 513798911..f68e4a029 100644 --- a/test/pleroma/user/backup_test.exs +++ b/test/pleroma/user/backup_test.exs @@ -82,7 +82,7 @@ test "it removes outdated backups after creating a fresh one" do assert {:ok, job1} = Backup.create(user) - assert {:ok, %Backup{id: backup1_id}} = ObanHelpers.perform(job1) + assert {:ok, %Backup{}} = ObanHelpers.perform(job1) assert {:ok, job2} = Backup.create(user) assert Pleroma.Repo.aggregate(Backup, :count) == 2 assert {:ok, backup2} = ObanHelpers.perform(job2) From da4a1e57b11d5600788b90f21d18bbcd97f6849f Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 28 Oct 2020 19:09:38 +0300 Subject: [PATCH 69/75] @doc fix. --- lib/pleroma/web/static_fe/static_fe_controller.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index 76b82589f..bdec0897a 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do @page_keys ["max_id", "min_id", "limit", "since_id", "order"] - @doc "Renders requested local public activity" + @doc "Renders requested local public activity or public activities of requested user" def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do with %Activity{local: true} = activity <- Activity.get_by_id_with_object(notice_id), @@ -46,7 +46,6 @@ def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do end end - @doc "Renders public activities of requested user" def show(%{assigns: %{username_or_id: username_or_id}} = conn, params) do with {_, %User{local: true} = user} <- {:fetch_user, User.get_cached_by_nickname_or_id(username_or_id)}, From 9f5f7dc9f956359204fc44a0627e20fd9765d8bd Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 28 Oct 2020 22:29:52 +0300 Subject: [PATCH 70/75] Fixed User.is_discoverable attribute rendering in Admin API User view. --- lib/pleroma/web/admin_api/views/account_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex index bda7ea19c..8bac24d3e 100644 --- a/lib/pleroma/web/admin_api/views/account_view.ex +++ b/lib/pleroma/web/admin_api/views/account_view.ex @@ -52,7 +52,7 @@ def render("credentials.json", %{user: user, for: for_user}) do :skip_thread_containment, :pleroma_settings_store, :raw_fields, - :discoverable, + :is_discoverable, :actor_type ]) |> Map.merge(%{ From 89c356d19fde3349a741522e7940a281487d9b08 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 29 Oct 2020 14:22:07 -0500 Subject: [PATCH 71/75] Improve Keyword descriptions for AdminFE --- config/description.exs | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/config/description.exs b/config/description.exs index 0bfa9979f..798cbe2ad 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1757,28 +1757,37 @@ related_policy: "Pleroma.Web.ActivityPub.MRF.KeywordPolicy", label: "MRF Keyword", type: :group, - description: "Reject or Word-Replace messages with a keyword or regex", + description: + "Reject or Word-Replace messages matching a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html).", children: [ %{ key: :reject, type: {:list, :string}, - description: - "A list of patterns which result in message being rejected. Each pattern can be a string or a regular expression.", + description: """ + A list of patterns which result in message being rejected. + + Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. + """, suggestions: ["foo", ~r/foo/iu] }, %{ key: :federated_timeline_removal, type: {:list, :string}, - description: - "A list of patterns which result in message being removed from federated timelines (a.k.a unlisted). Each pattern can be a string or a regular expression.", + description: """ + A list of patterns which result in message being removed from federated timelines (a.k.a unlisted). + + Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. + """, suggestions: ["foo", ~r/foo/iu] }, %{ key: :replace, type: {:list, :tuple}, - description: - "A list of tuples containing {pattern, replacement}. Each pattern can be a string or a regular expression.", - suggestions: [{"foo", "bar"}, {~r/foo/iu, "bar"}] + description: """ + **Pattern**: a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. + + **Replacement**: a string. Leaving the field empty is permitted. + """ } ] }, From 1a98476f48db380b5d26b882dad7fba745b3f160 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 30 Oct 2020 16:39:14 +0400 Subject: [PATCH 72/75] Remove unused aliases --- .../web/admin_api/controllers/admin_api_controller_test.exs | 1 - test/pleroma/web/admin_api/controllers/chat_controller_test.exs | 1 - .../admin_api/controllers/instance_document_controller_test.exs | 1 - .../web/admin_api/controllers/o_auth_app_controller_test.exs | 1 - test/pleroma/web/admin_api/controllers/relay_controller_test.exs | 1 - .../pleroma/web/admin_api/controllers/report_controller_test.exs | 1 - .../pleroma/web/admin_api/controllers/status_controller_test.exs | 1 - test/pleroma/web/admin_api/controllers/user_controller_test.exs | 1 - .../web/mastodon_api/controllers/timeline_controller_test.exs | 1 - .../web/pleroma_api/controllers/user_import_controller_test.exs | 1 - test/pleroma/web/plugs/http_security_plug_test.exs | 1 - test/pleroma/web/twitter_api/remote_follow_controller_test.exs | 1 - 12 files changed, 12 deletions(-) diff --git a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs index 34b26dddf..e2d6d5f6e 100644 --- a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs @@ -11,7 +11,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do import Swoosh.TestAssertions alias Pleroma.Activity - alias Pleroma.Config alias Pleroma.MFA alias Pleroma.ModerationLog alias Pleroma.Repo diff --git a/test/pleroma/web/admin_api/controllers/chat_controller_test.exs b/test/pleroma/web/admin_api/controllers/chat_controller_test.exs index bd4c9c9d1..5aefa1e60 100644 --- a/test/pleroma/web/admin_api/controllers/chat_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/chat_controller_test.exs @@ -9,7 +9,6 @@ defmodule Pleroma.Web.AdminAPI.ChatControllerTest do alias Pleroma.Chat alias Pleroma.Chat.MessageReference - alias Pleroma.Config alias Pleroma.ModerationLog alias Pleroma.Object alias Pleroma.Repo diff --git a/test/pleroma/web/admin_api/controllers/instance_document_controller_test.exs b/test/pleroma/web/admin_api/controllers/instance_document_controller_test.exs index 5f7b042f6..ce867dd0e 100644 --- a/test/pleroma/web/admin_api/controllers/instance_document_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/instance_document_controller_test.exs @@ -5,7 +5,6 @@ defmodule Pleroma.Web.AdminAPI.InstanceDocumentControllerTest do use Pleroma.Web.ConnCase, async: true import Pleroma.Factory - alias Pleroma.Config @dir "test/tmp/instance_static" @default_instance_panel ~s(

Welcome to Pleroma!

) diff --git a/test/pleroma/web/admin_api/controllers/o_auth_app_controller_test.exs b/test/pleroma/web/admin_api/controllers/o_auth_app_controller_test.exs index ed7c4172c..f388375d1 100644 --- a/test/pleroma/web/admin_api/controllers/o_auth_app_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/o_auth_app_controller_test.exs @@ -8,7 +8,6 @@ defmodule Pleroma.Web.AdminAPI.OAuthAppControllerTest do import Pleroma.Factory - alias Pleroma.Config alias Pleroma.Web setup do diff --git a/test/pleroma/web/admin_api/controllers/relay_controller_test.exs b/test/pleroma/web/admin_api/controllers/relay_controller_test.exs index adadf2b5c..b4c5e7567 100644 --- a/test/pleroma/web/admin_api/controllers/relay_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/relay_controller_test.exs @@ -7,7 +7,6 @@ defmodule Pleroma.Web.AdminAPI.RelayControllerTest do import Pleroma.Factory - alias Pleroma.Config alias Pleroma.ModerationLog alias Pleroma.Repo alias Pleroma.User diff --git a/test/pleroma/web/admin_api/controllers/report_controller_test.exs b/test/pleroma/web/admin_api/controllers/report_controller_test.exs index 57946e6bb..fa746d6ea 100644 --- a/test/pleroma/web/admin_api/controllers/report_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/report_controller_test.exs @@ -8,7 +8,6 @@ defmodule Pleroma.Web.AdminAPI.ReportControllerTest do import Pleroma.Factory alias Pleroma.Activity - alias Pleroma.Config alias Pleroma.ModerationLog alias Pleroma.Repo alias Pleroma.ReportNote diff --git a/test/pleroma/web/admin_api/controllers/status_controller_test.exs b/test/pleroma/web/admin_api/controllers/status_controller_test.exs index eff78fb0a..a18ef9e4b 100644 --- a/test/pleroma/web/admin_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/status_controller_test.exs @@ -8,7 +8,6 @@ defmodule Pleroma.Web.AdminAPI.StatusControllerTest do import Pleroma.Factory alias Pleroma.Activity - alias Pleroma.Config alias Pleroma.ModerationLog alias Pleroma.Repo alias Pleroma.User diff --git a/test/pleroma/web/admin_api/controllers/user_controller_test.exs b/test/pleroma/web/admin_api/controllers/user_controller_test.exs index da26caf25..5705306c7 100644 --- a/test/pleroma/web/admin_api/controllers/user_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/user_controller_test.exs @@ -9,7 +9,6 @@ defmodule Pleroma.Web.AdminAPI.UserControllerTest do import Mock import Pleroma.Factory - alias Pleroma.Config alias Pleroma.HTML alias Pleroma.ModerationLog alias Pleroma.Repo diff --git a/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs index c6e0268fd..9f1ee0424 100644 --- a/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs @@ -8,7 +8,6 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do import Pleroma.Factory import Tesla.Mock - alias Pleroma.Config alias Pleroma.User alias Pleroma.Web.CommonAPI diff --git a/test/pleroma/web/pleroma_api/controllers/user_import_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/user_import_controller_test.exs index 433c97e81..68723de71 100644 --- a/test/pleroma/web/pleroma_api/controllers/user_import_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/user_import_controller_test.exs @@ -6,7 +6,6 @@ defmodule Pleroma.Web.PleromaAPI.UserImportControllerTest do use Pleroma.Web.ConnCase use Oban.Testing, repo: Pleroma.Repo - alias Pleroma.Config alias Pleroma.Tests.ObanHelpers import Pleroma.Factory diff --git a/test/pleroma/web/plugs/http_security_plug_test.exs b/test/pleroma/web/plugs/http_security_plug_test.exs index 2297e3dac..df2b5ebb3 100644 --- a/test/pleroma/web/plugs/http_security_plug_test.exs +++ b/test/pleroma/web/plugs/http_security_plug_test.exs @@ -5,7 +5,6 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do use Pleroma.Web.ConnCase - alias Pleroma.Config alias Plug.Conn describe "http security enabled" do diff --git a/test/pleroma/web/twitter_api/remote_follow_controller_test.exs b/test/pleroma/web/twitter_api/remote_follow_controller_test.exs index 3852c7ce9..a3e784d13 100644 --- a/test/pleroma/web/twitter_api/remote_follow_controller_test.exs +++ b/test/pleroma/web/twitter_api/remote_follow_controller_test.exs @@ -5,7 +5,6 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do use Pleroma.Web.ConnCase - alias Pleroma.Config alias Pleroma.MFA alias Pleroma.MFA.TOTP alias Pleroma.User From d1698267a27bd5084916f5f6f36d66b1ff2ffc5f Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 31 Oct 2020 00:26:11 +0400 Subject: [PATCH 73/75] Fix credo warning --- lib/pleroma/web/router.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 9592d0f38..efe67ad7a 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -148,7 +148,7 @@ defmodule Pleroma.Web.Router do scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do pipe_through(:admin_api) - + put("/users/disable_mfa", AdminAPIController, :disable_mfa) put("/users/tag", AdminAPIController, :tag_users) delete("/users/tag", AdminAPIController, :untag_users) From 8e41baff40555ef7c74c8842d6fbfebc2368631a Mon Sep 17 00:00:00 2001 From: eugenijm Date: Sat, 31 Oct 2020 05:50:48 +0300 Subject: [PATCH 74/75] Add idempotency_key to the chat_message entity. --- CHANGELOG.md | 1 + docs/API/chats.md | 5 ++++- lib/pleroma/application.ex | 9 ++++++++- lib/pleroma/web/activity_pub/side_effects.ex | 6 ++++++ lib/pleroma/web/common_api.ex | 3 ++- .../web/pleroma_api/controllers/chat_controller.ex | 10 +++++++++- .../pleroma_api/views/chat/message_reference_view.ex | 11 +++++++++++ .../pleroma_api/controllers/chat_controller_test.exs | 2 ++ .../views/chat_message_reference_view_test.exs | 5 ++++- test/pleroma/web/streamer_test.exs | 4 +++- 10 files changed, 50 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11820d313..bb02d7b32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Pleroma API: Pagination for remote/local packs and emoji. - Admin API: (`GET /api/pleroma/admin/users`) added filters user by `unconfirmed` status - Admin API: (`GET /api/pleroma/admin/users`) added filters user by `actor_type` +- Pleroma API: Add `idempotency_key` to the chat message entity that can be used for optimistic message sending. diff --git a/docs/API/chats.md b/docs/API/chats.md index aa6119670..9857aac67 100644 --- a/docs/API/chats.md +++ b/docs/API/chats.md @@ -173,11 +173,14 @@ Returned data: "created_at": "2020-04-21T15:06:45.000Z", "emojis": [], "id": "12", - "unread": false + "unread": false, + "idempotency_key": "75442486-0874-440c-9db1-a7006c25a31f" } ] ``` +- idempotency_key: The copy of the `idempotency-key` HTTP request header that can be used for optimistic message sending. Included only during the first few minutes after the message creation. + ### Posting a chat message Posting a chat message for given Chat id works like this: diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 51e9dda3b..7c4cd9626 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -168,7 +168,11 @@ defp cachex_children do build_cachex("web_resp", limit: 2500), build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), build_cachex("failed_proxy_url", limit: 2500), - build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000) + build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000), + build_cachex("chat_message_id_idempotency_key", + expiration: chat_message_id_idempotency_key_expiration(), + limit: 500_000 + ) ] end @@ -178,6 +182,9 @@ defp emoji_packs_expiration, defp idempotency_expiration, do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60)) + defp chat_message_id_idempotency_key_expiration, + do: expiration(default: :timer.minutes(2), interval: :timer.seconds(60)) + defp seconds_valid_interval, do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid])) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 0fff5faf2..d552e91fc 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -312,6 +312,12 @@ def handle_object_creation(%{"type" => "ChatMessage"} = object, meta) do {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id) {:ok, cm_ref} = MessageReference.create(chat, object, user.ap_id != actor.ap_id) + Cachex.put( + :chat_message_id_idempotency_key_cache, + cm_ref.id, + meta[:idempotency_key] + ) + { ["user", "user:pleroma_chat"], {user, %{cm_ref | chat: chat, object: object}} diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 60a50b027..318ffc5d0 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -45,7 +45,8 @@ def post_chat_message(%User{} = user, %User{} = recipient, content, opts \\ []) {_, {:ok, %Activity{} = activity, _meta}} <- {:common_pipeline, Pipeline.common_pipeline(create_activity_data, - local: true + local: true, + idempotency_key: opts[:idempotency_key] )} do {:ok, activity} else diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex index 6357148d0..2c4d3f135 100644 --- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex @@ -80,7 +80,8 @@ def post_chat_message( %User{} = recipient <- User.get_cached_by_ap_id(chat.recipient), {:ok, activity} <- CommonAPI.post_chat_message(user, recipient, params[:content], - media_id: params[:media_id] + media_id: params[:media_id], + idempotency_key: idempotency_key(conn) ), message <- Object.normalize(activity, false), cm_ref <- MessageReference.for_chat_and_object(chat, message) do @@ -169,4 +170,11 @@ def show(%{assigns: %{user: user}} = conn, %{id: id}) do |> render("show.json", chat: chat) end end + + defp idempotency_key(conn) do + case get_req_header(conn, "idempotency-key") do + [key] -> key + _ -> nil + end + end end diff --git a/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex b/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex index d4e08b50d..c058fb340 100644 --- a/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex +++ b/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.PleromaAPI.Chat.MessageReferenceView do use Pleroma.Web, :view + alias Pleroma.Maps alias Pleroma.User alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.StatusView @@ -37,6 +38,7 @@ def render( Pleroma.Web.RichMedia.Helpers.fetch_data_for_object(object) ) } + |> put_idempotency_key() end def render("index.json", opts) do @@ -47,4 +49,13 @@ def render("index.json", opts) do Map.put(opts, :as, :chat_message_reference) ) end + + defp put_idempotency_key(data) do + with {:ok, idempotency_key} <- Cachex.get(:chat_message_id_idempotency_key_cache, data.id) do + data + |> Maps.put_if_present(:idempotency_key, idempotency_key) + else + _ -> data + end + end end diff --git a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs index 6381f9757..fa6b9db65 100644 --- a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs @@ -82,11 +82,13 @@ test "it posts a message to the chat", %{conn: conn, user: user} do result = conn |> put_req_header("content-type", "application/json") + |> put_req_header("idempotency-key", "123") |> post("/api/v1/pleroma/chats/#{chat.id}/messages", %{"content" => "Hallo!!"}) |> json_response_and_validate_schema(200) assert result["content"] == "Hallo!!" assert result["chat_id"] == chat.id |> to_string() + assert result["idempotency_key"] == "123" end test "it fails if there is no content", %{conn: conn, user: user} do diff --git a/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs b/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs index f171a1e55..ae8257870 100644 --- a/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs +++ b/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs @@ -25,7 +25,9 @@ test "it displays a chat message" do } {:ok, upload} = ActivityPub.upload(file, actor: user.ap_id) - {:ok, activity} = CommonAPI.post_chat_message(user, recipient, "kippis :firefox:") + + {:ok, activity} = + CommonAPI.post_chat_message(user, recipient, "kippis :firefox:", idempotency_key: "123") chat = Chat.get(user.id, recipient.ap_id) @@ -42,6 +44,7 @@ test "it displays a chat message" do assert chat_message[:created_at] assert chat_message[:unread] == false assert match?([%{shortcode: "firefox"}], chat_message[:emojis]) + assert chat_message[:idempotency_key] == "123" clear_config([:rich_media, :enabled], true) diff --git a/test/pleroma/web/streamer_test.exs b/test/pleroma/web/streamer_test.exs index 185724a9f..395016da2 100644 --- a/test/pleroma/web/streamer_test.exs +++ b/test/pleroma/web/streamer_test.exs @@ -255,7 +255,9 @@ test "it sends chat messages to the 'user:pleroma_chat' stream", %{ } do other_user = insert(:user) - {:ok, create_activity} = CommonAPI.post_chat_message(other_user, user, "hey cirno") + {:ok, create_activity} = + CommonAPI.post_chat_message(other_user, user, "hey cirno", idempotency_key: "123") + object = Object.normalize(create_activity, false) chat = Chat.get(user.id, other_user.ap_id) cm_ref = MessageReference.for_chat_and_object(chat, object) From 8f00d90f9199e384fb1befb677c1c0595a0c854c Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 1 Nov 2020 12:05:39 +0300 Subject: [PATCH 75/75] Use Pleroma.HTTP instead of Tesla Closes #2275 As discovered in the issue, captcha used Tesla.get instead of Pleroma.HTTP. I've also grep'ed the repo and changed the other place where this was used. --- lib/pleroma/captcha/kocaptcha.ex | 2 +- lib/pleroma/emoji/pack.ex | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/captcha/kocaptcha.ex b/lib/pleroma/captcha/kocaptcha.ex index 337506647..201b55ab4 100644 --- a/lib/pleroma/captcha/kocaptcha.ex +++ b/lib/pleroma/captcha/kocaptcha.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Captcha.Kocaptcha do def new do endpoint = Pleroma.Config.get!([__MODULE__, :endpoint]) - case Tesla.get(endpoint <> "/new") do + case Pleroma.HTTP.get(endpoint <> "/new") do {:error, _} -> %{error: :kocaptcha_service_unavailable} diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index 0670f29f1..ca58e5432 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -594,7 +594,7 @@ defp fetch_pack_info(remote_pack, uri, name) do end defp download_archive(url, sha) do - with {:ok, %{body: archive}} <- Tesla.get(url) do + with {:ok, %{body: archive}} <- Pleroma.HTTP.get(url) do if Base.decode16!(sha) == :crypto.hash(:sha256, archive) do {:ok, archive} else @@ -617,7 +617,7 @@ defp fallback_sha_changed?(pack, data) do end defp update_sha_and_save_metadata(pack, data) do - with {:ok, %{body: zip}} <- Tesla.get(data[:"fallback-src"]), + with {:ok, %{body: zip}} <- Pleroma.HTTP.get(data[:"fallback-src"]), :ok <- validate_has_all_files(pack, zip) do fallback_sha = :sha256 |> :crypto.hash(zip) |> Base.encode16()