From afa8b469ed0a71247f27efec08d6eeac24b6674f Mon Sep 17 00:00:00 2001 From: Alex Gleason <alex@alexgleason.me> Date: Wed, 1 Jul 2020 21:12:59 -0500 Subject: [PATCH 01/74] Allow restricting public timeline by instance --- lib/pleroma/web/activity_pub/activity_pub.ex | 12 ++---------- .../web/api_spec/operations/timeline_operation.ex | 10 ++++++++++ .../mastodon_api/controllers/timeline_controller.ex | 1 + .../controllers/timeline_controller_test.exs | 12 ++++++++++++ 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index bc7b5d95a..9ce2b04dd 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -927,16 +927,8 @@ defp restrict_muted_reblogs(query, %{muting_user: %User{} = user} = opts) do defp restrict_muted_reblogs(query, _), do: query - defp restrict_instance(query, %{instance: instance}) do - users = - from( - u in User, - select: u.ap_id, - where: fragment("? LIKE ?", u.nickname, ^"%@#{instance}") - ) - |> Repo.all() - - from(activity in query, where: activity.actor in ^users) + defp restrict_instance(query, %{instance: instance}) when is_binary(instance) do + from(activity in query, where: ilike(activity.actor, ^"%://#{instance}/%")) end defp restrict_instance(query, _), do: query diff --git a/lib/pleroma/web/api_spec/operations/timeline_operation.ex b/lib/pleroma/web/api_spec/operations/timeline_operation.ex index 8e19bace7..83cdbad69 100644 --- a/lib/pleroma/web/api_spec/operations/timeline_operation.ex +++ b/lib/pleroma/web/api_spec/operations/timeline_operation.ex @@ -59,6 +59,7 @@ def public_operation do security: [%{"oAuth" => ["read:statuses"]}], parameters: [ local_param(), + instance_param(), only_media_param(), with_muted_param(), exclude_visibilities_param(), @@ -158,6 +159,15 @@ defp local_param do ) end + defp instance_param do + Operation.parameter( + :instance, + :query, + %Schema{type: :string}, + "Show only statuses from the given domain" + ) + end + defp with_muted_param do Operation.parameter(:with_muted, :query, BooleanLike, "Includeactivities by muted users") end diff --git a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex index ab7b1d6aa..7dccc0005 100644 --- a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex @@ -110,6 +110,7 @@ def public(%{assigns: %{user: user}} = conn, params) do |> Map.put(:blocking_user, user) |> Map.put(:muting_user, user) |> Map.put(:reply_filtering_user, user) + |> Map.put(:instance, params[:instance]) |> ActivityPub.fetch_public_activities() conn diff --git a/test/web/mastodon_api/controllers/timeline_controller_test.exs b/test/web/mastodon_api/controllers/timeline_controller_test.exs index 50e0d783d..6acd512c7 100644 --- a/test/web/mastodon_api/controllers/timeline_controller_test.exs +++ b/test/web/mastodon_api/controllers/timeline_controller_test.exs @@ -140,6 +140,18 @@ test "doesn't return replies if follow is posting with users from blocked domain activities = json_response_and_validate_schema(res_conn, 200) [%{"id" => ^activity_id}] = activities end + + test "can be filtered by instance", %{conn: conn} do + user = insert(:user, ap_id: "https://lain.com/users/lain") + insert(:note_activity, local: false) + insert(:note_activity, local: false) + + {:ok, _} = CommonAPI.post(user, %{status: "test"}) + + conn = get(conn, "/api/v1/timelines/public?instance=lain.com") + + assert length(json_response_and_validate_schema(conn, :ok)) == 1 + end end defp local_and_remote_activities do From e9cff69bceb5cfb14a13ef09d1d7ab1079028a58 Mon Sep 17 00:00:00 2001 From: Alex Gleason <alex@alexgleason.me> Date: Sun, 2 Aug 2020 12:24:40 -0500 Subject: [PATCH 02/74] Add TagPolicy as default MRF, #2010 --- config/config.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 134dccf49..68a050315 100644 --- a/config/config.exs +++ b/config/config.exs @@ -719,7 +719,7 @@ config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: false config :pleroma, :mrf, - policies: Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy, + policies: [Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy, Pleroma.Web.ActivityPub.MRF.TagPolicy], transparency: true, transparency_exclusions: [] From ad9c925efb77287316f5dbac26f6a1b16662910a Mon Sep 17 00:00:00 2001 From: Alex Gleason <alex@alexgleason.me> Date: Wed, 5 Aug 2020 13:08:13 -0500 Subject: [PATCH 03/74] Speed up instance timeline query --- lib/pleroma/web/activity_pub/activity_pub.ex | 5 ++++- test/web/admin_api/controllers/admin_api_controller_test.exs | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 9ce2b04dd..76fc9c3ee 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -928,7 +928,10 @@ defp restrict_muted_reblogs(query, %{muting_user: %User{} = user} = opts) do defp restrict_muted_reblogs(query, _), do: query defp restrict_instance(query, %{instance: instance}) when is_binary(instance) do - from(activity in query, where: ilike(activity.actor, ^"%://#{instance}/%")) + from( + activity in query, + where: fragment("split_part(actor::text, '/'::text, 3) = ?", ^instance) + ) end defp restrict_instance(query, _), do: query diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs index da91cd552..26194eb81 100644 --- a/test/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/web/admin_api/controllers/admin_api_controller_test.exs @@ -1647,8 +1647,8 @@ test "sets password_reset_pending to true", %{conn: conn} do describe "instances" do test "GET /instances/:instance/statuses", %{conn: conn} do - user = insert(:user, local: false, nickname: "archaeme@archae.me") - user2 = insert(:user, local: false, nickname: "test@test.com") + user = insert(:user, local: false, ap_id: "https://archae.me/users/archaeme") + user2 = insert(:user, local: false, ap_id: "https://test.com/users/test") insert_pair(:note_activity, user: user) activity = insert(:note_activity, user: user2) From 24ce9c011caf7401fb261c7df4196b2ef9ba3d90 Mon Sep 17 00:00:00 2001 From: Alex Gleason <alex@alexgleason.me> Date: Wed, 5 Aug 2020 19:33:51 +0000 Subject: [PATCH 04/74] Apply 1 suggestion(s) to 1 file(s) --- lib/pleroma/web/api_spec/operations/timeline_operation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/operations/timeline_operation.ex b/lib/pleroma/web/api_spec/operations/timeline_operation.ex index 83cdbad69..95720df9f 100644 --- a/lib/pleroma/web/api_spec/operations/timeline_operation.ex +++ b/lib/pleroma/web/api_spec/operations/timeline_operation.ex @@ -169,7 +169,7 @@ defp instance_param do end defp with_muted_param do - Operation.parameter(:with_muted, :query, BooleanLike, "Includeactivities by muted users") + Operation.parameter(:with_muted, :query, BooleanLike, "Include activities by muted users") end defp exclude_visibilities_param do From c56e3d4f3bfb090d19bdbe93dac6cede7616cc4d Mon Sep 17 00:00:00 2001 From: Roman Chvanikov <chvanikoff@pm.me> Date: Tue, 8 Sep 2020 13:26:44 +0300 Subject: [PATCH 05/74] Add expires_in param for account mutes --- config/config.exs | 5 ++- lib/pleroma/user.ex | 45 ++++++++++--------- .../api_spec/operations/account_operation.ex | 15 ++++++- .../controllers/account_controller.ex | 2 +- lib/pleroma/workers/mute_expire_worker.ex | 22 +++++++++ test/notification_test.exs | 4 +- test/user_test.exs | 2 +- .../notification_controller_test.exs | 2 +- .../mastodon_api/views/account_view_test.exs | 2 +- 9 files changed, 69 insertions(+), 30 deletions(-) create mode 100644 lib/pleroma/workers/mute_expire_worker.ex diff --git a/config/config.exs b/config/config.exs index ed37b93c0..0649f3078 100644 --- a/config/config.exs +++ b/config/config.exs @@ -541,7 +541,8 @@ background: 5, remote_fetcher: 2, attachments_cleanup: 5, - new_users_digest: 1 + new_users_digest: 1, + mute_expire: 5 ], plugins: [Oban.Plugins.Pruner], crontab: [ @@ -672,7 +673,7 @@ # With no frontend configuration, the bundled files from the `static` directory will # be used. # -# config :pleroma, :frontends, +# config :pleroma, :frontends, # primary: %{"name" => "pleroma-fe", "ref" => "develop"}, # admin: %{"name" => "admin-fe", "ref" => "stable"}, # available: %{...} diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 94c96de8d..040db8d80 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1356,14 +1356,34 @@ def get_recipients_from_activity(%Activity{recipients: to, actor: actor}) do |> Repo.all() end - @spec mute(User.t(), User.t(), boolean()) :: + @spec mute(User.t(), User.t(), map()) :: {:ok, list(UserRelationship.t())} | {:error, String.t()} - def mute(%User{} = muter, %User{} = mutee, notifications? \\ true) do - add_to_mutes(muter, mutee, notifications?) + def mute(%User{} = muter, %User{} = mutee, params \\ %{}) do + notifications? = Map.get(params, :notifications, true) + expires_in = Map.get(params, :expires_in, 0) + + with {:ok, user_mute} <- UserRelationship.create_mute(muter, mutee), + {:ok, user_notification_mute} <- + (notifications? && UserRelationship.create_notification_mute(muter, mutee)) || + {:ok, nil} do + with seconds when seconds > 0 <- expires_in do + Pleroma.Workers.MuteExpireWorker.enqueue( + "unmute", + %{"muter" => muter.id, "mutee" => mutee.id}, + schedule_in: expires_in + ) + end + + {:ok, Enum.filter([user_mute, user_notification_mute], & &1)} + end end def unmute(%User{} = muter, %User{} = mutee) do - remove_from_mutes(muter, mutee) + with {:ok, user_mute} <- UserRelationship.delete_mute(muter, mutee), + {:ok, user_notification_mute} <- + UserRelationship.delete_notification_mute(muter, mutee) do + {:ok, [user_mute, user_notification_mute]} + end end def subscribe(%User{} = subscriber, %User{} = target) do @@ -2379,23 +2399,6 @@ defp remove_from_block(%User{} = user, %User{} = blocked) do UserRelationship.delete_block(user, blocked) end - defp add_to_mutes(%User{} = user, %User{} = muted_user, notifications?) do - with {:ok, user_mute} <- UserRelationship.create_mute(user, muted_user), - {:ok, user_notification_mute} <- - (notifications? && UserRelationship.create_notification_mute(user, muted_user)) || - {:ok, nil} do - {:ok, Enum.filter([user_mute, user_notification_mute], & &1)} - end - end - - defp remove_from_mutes(user, %User{} = muted_user) do - with {:ok, user_mute} <- UserRelationship.delete_mute(user, muted_user), - {:ok, user_notification_mute} <- - UserRelationship.delete_notification_mute(user, muted_user) do - {:ok, [user_mute, user_notification_mute]} - end - end - def set_invisible(user, invisible) do params = %{invisible: invisible} diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index aaebc9b5c..de715a077 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -262,6 +262,12 @@ def mute_operation do :query, %Schema{allOf: [BooleanLike], default: true}, "Mute notifications in addition to statuses? Defaults to `true`." + ), + Operation.parameter( + :expires_in, + :query, + %Schema{type: :integer, default: 0}, + "Expire the mute in `expires_in` seconds. Default 0 for infinity" ) ], responses: %{ @@ -718,10 +724,17 @@ defp mute_request do nullable: true, description: "Mute notifications in addition to statuses? Defaults to true.", default: true + }, + expires_in: %Schema{ + type: :integer, + nullable: true, + description: "Expire the mute in `expires_in` seconds. Default 0 for infinity", + default: 0 } }, example: %{ - "notifications" => true + "notifications" => true, + "expires_in" => 86_400 } } end diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 95d8452df..ca1a79f5e 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -394,7 +394,7 @@ def unfollow(%{assigns: %{user: follower, account: followed}} = conn, _params) d @doc "POST /api/v1/accounts/:id/mute" def mute(%{assigns: %{user: muter, account: muted}, body_params: params} = conn, _params) do - with {:ok, _user_relationships} <- User.mute(muter, muted, params.notifications) do + with {:ok, _user_relationships} <- User.mute(muter, muted, params) do render(conn, "relationship.json", user: muter, target: muted) else {:error, message} -> json_response(conn, :forbidden, %{error: message}) diff --git a/lib/pleroma/workers/mute_expire_worker.ex b/lib/pleroma/workers/mute_expire_worker.ex new file mode 100644 index 000000000..b8ec939a9 --- /dev/null +++ b/lib/pleroma/workers/mute_expire_worker.ex @@ -0,0 +1,22 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.MuteExpireWorker do + use Pleroma.Workers.WorkerHelper, queue: "mute_expire" + + require Logger + + @impl Oban.Worker + def perform(%Job{args: %{"op" => "unmute", "muter" => muter_id, "mutee" => mutee_id}}) do + muter = Pleroma.User.get_by_id(muter_id) + mutee = Pleroma.User.get_by_id(mutee_id) + Pleroma.User.unmute(muter, mutee) + :ok + end + + def perform(any) do + Logger.error("Got call to perform(#{inspect(any)})") + :ok + end +end diff --git a/test/notification_test.exs b/test/notification_test.exs index a09b08675..ffd737969 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -227,7 +227,7 @@ test "notification created if user is muted without notifications" do muter = insert(:user) muted = insert(:user) - {:ok, _user_relationships} = User.mute(muter, muted, false) + {:ok, _user_relationships} = User.mute(muter, muted, %{notifications: false}) {:ok, activity} = CommonAPI.post(muted, %{status: "Hi @#{muter.nickname}"}) @@ -1013,7 +1013,7 @@ test "move activity generates a notification" do test "it returns notifications for muted user without notifications", %{user: user} do muted = insert(:user) - {:ok, _user_relationships} = User.mute(user, muted, false) + {:ok, _user_relationships} = User.mute(user, muted, %{notifications: false}) {:ok, _activity} = CommonAPI.post(muted, %{status: "hey @#{user.nickname}"}) diff --git a/test/user_test.exs b/test/user_test.exs index 50f72549e..b23e36be3 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -981,7 +981,7 @@ test "it mutes user without notifications" do refute User.mutes?(user, muted_user) refute User.muted_notifications?(user, muted_user) - {:ok, _user_relationships} = User.mute(user, muted_user, false) + {:ok, _user_relationships} = User.mute(user, muted_user, %{notifications: false}) assert User.mutes?(user, muted_user) refute User.muted_notifications?(user, muted_user) diff --git a/test/web/mastodon_api/controllers/notification_controller_test.exs b/test/web/mastodon_api/controllers/notification_controller_test.exs index 70ef0e8b5..5fd518c60 100644 --- a/test/web/mastodon_api/controllers/notification_controller_test.exs +++ b/test/web/mastodon_api/controllers/notification_controller_test.exs @@ -502,7 +502,7 @@ test "see notifications after muting user without notifications" do assert length(json_response_and_validate_schema(ret_conn, 200)) == 1 - {:ok, _user_relationships} = User.mute(user, user2, false) + {:ok, _user_relationships} = User.mute(user, user2, %{notifications: false}) conn = get(conn, "/api/v1/notifications") diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs index 8f37efa3c..c34cbcfc1 100644 --- a/test/web/mastodon_api/views/account_view_test.exs +++ b/test/web/mastodon_api/views/account_view_test.exs @@ -277,7 +277,7 @@ test "represent a relationship for the following and followed user" do {:ok, user} = User.follow(user, other_user) {:ok, other_user} = User.follow(other_user, user) {:ok, _subscription} = User.subscribe(user, other_user) - {:ok, _user_relationships} = User.mute(user, other_user, true) + {:ok, _user_relationships} = User.mute(user, other_user, %{notifications: true}) {:ok, _reblog_mute} = CommonAPI.hide_reblogs(user, other_user) expected = From f6b250fb8d13f6788c1ecc6c1287e76febbfd888 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov <chvanikoff@pm.me> Date: Tue, 8 Sep 2020 14:11:00 +0300 Subject: [PATCH 06/74] Add test for expiring mutes --- test/user_test.exs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/user_test.exs b/test/user_test.exs index b23e36be3..83c017ec5 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -963,6 +963,19 @@ test "it mutes people" do assert User.muted_notifications?(user, muted_user) end + test "expiring" do + user = insert(:user) + muted_user = insert(:user) + + {:ok, _user_relationships} = User.mute(user, muted_user, %{expires_in: 60}) + assert User.mutes?(user, muted_user) + + assert_enqueued( + worker: Pleroma.Workers.MuteExpireWorker, + args: %{"op" => "unmute", "muter" => user.id, "mutee" => muted_user.id} + ) + end + test "it unmutes users" do user = insert(:user) muted_user = insert(:user) From e3f845b24363cd867ab85b7297f2d34bfa16b13f Mon Sep 17 00:00:00 2001 From: Roman Chvanikov <chvanikoff@pm.me> Date: Tue, 8 Sep 2020 15:13:50 +0300 Subject: [PATCH 07/74] Add expiring mutes for activities --- lib/pleroma/user.ex | 6 ++--- .../api_spec/operations/status_operation.ex | 22 ++++++++++++++++++- lib/pleroma/web/common_api/common_api.ex | 12 +++++++++- lib/pleroma/workers/mute_expire_worker.ex | 10 ++++++--- test/user_test.exs | 12 ++++++++-- test/web/common_api/common_api_test.exs | 18 +++++++++++++++ 6 files changed, 70 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 040db8d80..46e03553c 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1366,10 +1366,10 @@ def mute(%User{} = muter, %User{} = mutee, params \\ %{}) do {:ok, user_notification_mute} <- (notifications? && UserRelationship.create_notification_mute(muter, mutee)) || {:ok, nil} do - with seconds when seconds > 0 <- expires_in do + if expires_in > 0 do Pleroma.Workers.MuteExpireWorker.enqueue( - "unmute", - %{"muter" => muter.id, "mutee" => mutee.id}, + "unmute_user", + %{"muter_id" => muter.id, "mutee_id" => mutee.id}, schedule_in: expires_in ) end diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 5bd4619d5..6589a16f3 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -223,7 +223,27 @@ def mute_conversation_operation do security: [%{"oAuth" => ["write:mutes"]}], description: "Do not receive notifications for the thread that this status is part of.", operationId: "StatusController.mute_conversation", - parameters: [id_param()], + requestBody: + request_body("Parameters", %Schema{ + type: :object, + properties: %{ + expires_in: %Schema{ + type: :integer, + nullable: true, + description: "Expire the mute in `expires_in` seconds. Default 0 for infinity", + default: 0 + } + } + }), + parameters: [ + id_param(), + Operation.parameter( + :expires_in, + :query, + %Schema{type: :integer, default: 0}, + "Expire the mute in `expires_in` seconds. Default 0 for infinity" + ) + ], responses: %{ 200 => status_response(), 400 => Operation.response("Error", "application/json", ApiError) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 4ab533658..b217c4d10 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -451,9 +451,19 @@ def unpin(id, user) do end end - def add_mute(user, activity) do + def add_mute(user, activity, params \\ %{}) do + expires_in = Map.get(params, :expires_in, 0) + with {:ok, _} <- ThreadMute.add_mute(user.id, activity.data["context"]), _ <- Pleroma.Notification.mark_context_as_read(user, activity.data["context"]) do + if expires_in > 0 do + Pleroma.Workers.MuteExpireWorker.enqueue( + "unmute_conversation", + %{"user_id" => user.id, "activity_id" => activity.id}, + schedule_in: expires_in + ) + end + {:ok, activity} else {:error, _} -> {:error, dgettext("errors", "conversation is already muted")} diff --git a/lib/pleroma/workers/mute_expire_worker.ex b/lib/pleroma/workers/mute_expire_worker.ex index b8ec939a9..622fdbadd 100644 --- a/lib/pleroma/workers/mute_expire_worker.ex +++ b/lib/pleroma/workers/mute_expire_worker.ex @@ -8,15 +8,19 @@ defmodule Pleroma.Workers.MuteExpireWorker do require Logger @impl Oban.Worker - def perform(%Job{args: %{"op" => "unmute", "muter" => muter_id, "mutee" => mutee_id}}) do + def perform(%Job{args: %{"op" => "unmute_user", "muter_id" => muter_id, "mutee_id" => mutee_id}}) do muter = Pleroma.User.get_by_id(muter_id) mutee = Pleroma.User.get_by_id(mutee_id) Pleroma.User.unmute(muter, mutee) :ok end - def perform(any) do - Logger.error("Got call to perform(#{inspect(any)})") + def perform(%Job{ + args: %{"op" => "unmute_conversation", "user_id" => user_id, "activity_id" => activity_id} + }) do + user = Pleroma.User.get_by_id(user_id) + activity = Pleroma.Activity.get_by_id(activity_id) + Pleroma.Web.CommonAPI.remove_mute(user, activity) :ok end end diff --git a/test/user_test.exs b/test/user_test.exs index 83c017ec5..d49afb35a 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -970,10 +970,18 @@ test "expiring" do {:ok, _user_relationships} = User.mute(user, muted_user, %{expires_in: 60}) assert User.mutes?(user, muted_user) + worker = Pleroma.Workers.MuteExpireWorker + args = %{"op" => "unmute_user", "muter_id" => user.id, "mutee_id" => muted_user.id} + assert_enqueued( - worker: Pleroma.Workers.MuteExpireWorker, - args: %{"op" => "unmute", "muter" => user.id, "mutee" => muted_user.id} + worker: worker, + args: args ) + + assert :ok = perform_job(worker, args) + + refute User.mutes?(user, muted_user) + refute User.muted_notifications?(user, muted_user) end test "it unmutes users" do diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 800db9a20..7ceb7ec7f 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -3,7 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.CommonAPITest do + use Oban.Testing, repo: Pleroma.Repo use Pleroma.DataCase + alias Pleroma.Activity alias Pleroma.Chat alias Pleroma.Conversation.Participation @@ -878,6 +880,22 @@ test "add mute", %{user: user, activity: activity} do assert CommonAPI.thread_muted?(user, activity) end + test "add expiring mute", %{user: user, activity: activity} do + {:ok, _} = CommonAPI.add_mute(user, activity, %{expires_in: 60}) + assert CommonAPI.thread_muted?(user, activity) + + worker = Pleroma.Workers.MuteExpireWorker + args = %{"op" => "unmute_conversation", "user_id" => user.id, "activity_id" => activity.id} + + assert_enqueued( + worker: worker, + args: args + ) + + assert :ok = perform_job(worker, args) + refute CommonAPI.thread_muted?(user, activity) + end + test "remove mute", %{user: user, activity: activity} do CommonAPI.add_mute(user, activity) {:ok, _} = CommonAPI.remove_mute(user, activity) From 91b9985e1c0ef1766eb1705364e1aebe69d9b9bd Mon Sep 17 00:00:00 2001 From: Roman Chvanikov <chvanikoff@pm.me> Date: Tue, 8 Sep 2020 15:26:06 +0300 Subject: [PATCH 08/74] Pass expires_in param from status controller --- lib/pleroma/web/mastodon_api/controllers/status_controller.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index ecfa38489..da14c0b6c 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -285,9 +285,9 @@ def unbookmark(%{assigns: %{user: user}} = conn, %{id: id}) do end @doc "POST /api/v1/statuses/:id/mute" - def mute_conversation(%{assigns: %{user: user}} = conn, %{id: id}) do + def mute_conversation(%{assigns: %{user: user}, body_params: params} = conn, %{id: id}) do with %Activity{} = activity <- Activity.get_by_id(id), - {:ok, activity} <- CommonAPI.add_mute(user, activity) do + {:ok, activity} <- CommonAPI.add_mute(user, activity, params) do try_render(conn, "show.json", activity: activity, for: user, as: :activity) end end From de2499e54b33a1746e5f6a5b79f1422d31c11570 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Wed, 9 Sep 2020 09:52:07 +0300 Subject: [PATCH 09/74] don't run update in tests --- lib/pleroma/stats.ex | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index e5c9c668b..48afe901e 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -23,7 +23,6 @@ def start_link(_) do @impl true def init(_args) do - if Pleroma.Config.get(:env) == :test, do: :ok = Ecto.Adapters.SQL.Sandbox.checkout(Repo) {:ok, nil, {:continue, :calculate_stats}} end @@ -32,11 +31,6 @@ def force_update do GenServer.call(__MODULE__, :force_update) end - @doc "Performs collect stats" - def do_collect do - GenServer.cast(__MODULE__, :run_update) - end - @doc "Returns stats data" @spec get_stats() :: %{ domain_count: non_neg_integer(), @@ -111,7 +105,11 @@ def get_status_visibility_count(instance \\ nil) do @impl true def handle_continue(:calculate_stats, _) do stats = calculate_stat_data() - Process.send_after(self(), :run_update, @interval) + + unless Pleroma.Config.get(:env) == :test do + Process.send_after(self(), :run_update, @interval) + end + {:noreply, stats} end @@ -126,13 +124,6 @@ def handle_call(:get_state, _from, state) do {:reply, state, state} end - @impl true - def handle_cast(:run_update, _state) do - new_stats = calculate_stat_data() - - {:noreply, new_stats} - end - @impl true def handle_info(:run_update, _) do new_stats = calculate_stat_data() From 527afb813af6c64337d02ddf1a2f159fe557acbc Mon Sep 17 00:00:00 2001 From: Roman Chvanikov <chvanikoff@pm.me> Date: Sun, 13 Sep 2020 12:23:45 +0300 Subject: [PATCH 10/74] Remove unused require --- lib/pleroma/workers/mute_expire_worker.ex | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/pleroma/workers/mute_expire_worker.ex b/lib/pleroma/workers/mute_expire_worker.ex index 622fdbadd..c8b44894e 100644 --- a/lib/pleroma/workers/mute_expire_worker.ex +++ b/lib/pleroma/workers/mute_expire_worker.ex @@ -5,8 +5,6 @@ defmodule Pleroma.Workers.MuteExpireWorker do use Pleroma.Workers.WorkerHelper, queue: "mute_expire" - require Logger - @impl Oban.Worker def perform(%Job{args: %{"op" => "unmute_user", "muter_id" => muter_id, "mutee_id" => mutee_id}}) do muter = Pleroma.User.get_by_id(muter_id) From 28d0986f839651df7d305da8932f7b5c48a4fbfb Mon Sep 17 00:00:00 2001 From: Roman Chvanikov <chvanikoff@pm.me> Date: Sun, 20 Sep 2020 20:51:20 +0300 Subject: [PATCH 11/74] Refactor mutes removing in CommonAPI and User --- lib/pleroma/user.ex | 14 ++++++++++++++ lib/pleroma/web/common_api/common_api.ex | 18 +++++++++++++++++- lib/pleroma/workers/mute_expire_worker.ex | 8 ++------ test/user_test.exs | 11 +++++++++++ test/web/common_api/common_api_test.exs | 6 ++++++ 5 files changed, 50 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 83eb4d5ff..83e89a12c 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1385,6 +1385,20 @@ def unmute(%User{} = muter, %User{} = mutee) do end end + def unmute(muter_id, mutee_id) do + with {:muter, %User{} = muter} <- {:muter, User.get_by_id(muter_id)}, + {:mutee, %User{} = mutee} <- {:mutee, User.get_by_id(mutee_id)} do + unmute(muter, mutee) + else + {who, result} = error -> + Logger.warn( + "User.unmute/2 failed. #{who}: #{result}, muter_id: #{muter_id}, mutee_id: #{mutee_id}" + ) + + {:error, error} + end + end + def subscribe(%User{} = subscriber, %User{} = target) do deny_follow_blocked = Config.get([:user, :deny_follow_blocked]) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index fca9246a5..aa4c6ddab 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -472,11 +472,27 @@ def add_mute(user, activity, params \\ %{}) do end end - def remove_mute(user, activity) do + def remove_mute(%User{} = user, %Activity{} = activity) do ThreadMute.remove_mute(user.id, activity.data["context"]) {:ok, activity} end + def remove_mute(user_id, activity_id) do + with {:user, %User{} = user} <- {:user, User.get_by_id(user_id)}, + {:activity, %Activity{} = activity} <- {:activity, Activity.get_by_id(activity_id)} do + remove_mute(user, activity) + else + {what, result} = error -> + Logger.warn( + "CommonAPI.remove_mute/2 failed. #{what}: #{result}, user_id: #{user_id}, activity_id: #{ + activity_id + }" + ) + + {:error, error} + end + end + def thread_muted?(%User{id: user_id}, %{data: %{"context" => context}}) when is_binary(context) do ThreadMute.exists?(user_id, context) diff --git a/lib/pleroma/workers/mute_expire_worker.ex b/lib/pleroma/workers/mute_expire_worker.ex index c8b44894e..32a12ba85 100644 --- a/lib/pleroma/workers/mute_expire_worker.ex +++ b/lib/pleroma/workers/mute_expire_worker.ex @@ -7,18 +7,14 @@ defmodule Pleroma.Workers.MuteExpireWorker do @impl Oban.Worker def perform(%Job{args: %{"op" => "unmute_user", "muter_id" => muter_id, "mutee_id" => mutee_id}}) do - muter = Pleroma.User.get_by_id(muter_id) - mutee = Pleroma.User.get_by_id(mutee_id) - Pleroma.User.unmute(muter, mutee) + Pleroma.User.unmute(muter_id, mutee_id) :ok end def perform(%Job{ args: %{"op" => "unmute_conversation", "user_id" => user_id, "activity_id" => activity_id} }) do - user = Pleroma.User.get_by_id(user_id) - activity = Pleroma.Activity.get_by_id(activity_id) - Pleroma.Web.CommonAPI.remove_mute(user, activity) + Pleroma.Web.CommonAPI.remove_mute(user_id, activity_id) :ok end end diff --git a/test/user_test.exs b/test/user_test.exs index ce0d4d38b..79c8b76b8 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1034,6 +1034,17 @@ test "it unmutes users" do refute User.muted_notifications?(user, muted_user) end + test "it unmutes users by id" do + user = insert(:user) + muted_user = insert(:user) + + {:ok, _user_relationships} = User.mute(user, muted_user) + {:ok, _user_mute} = User.unmute(user.id, muted_user.id) + + refute User.mutes?(user, muted_user) + refute User.muted_notifications?(user, muted_user) + end + test "it mutes user without notifications" do user = insert(:user) muted_user = insert(:user) diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index bf4353a57..45ab1e0bd 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -913,6 +913,12 @@ test "remove mute", %{user: user, activity: activity} do refute CommonAPI.thread_muted?(user, activity) end + test "remove mute by ids", %{user: user, activity: activity} do + CommonAPI.add_mute(user, activity) + {:ok, _} = CommonAPI.remove_mute(user.id, activity.id) + refute CommonAPI.thread_muted?(user, activity) + end + test "check that mutes can't be duplicate", %{user: user, activity: activity} do CommonAPI.add_mute(user, activity) {:error, _} = CommonAPI.add_mute(user, activity) From b0bd81ef7187ddf5b4e6cfbc1780fc60b65798c6 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov <chvanikoff@pm.me> Date: Sun, 20 Sep 2020 20:58:32 +0300 Subject: [PATCH 12/74] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a7e27fd3..9fd5f9efe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - Experimental websocket-based federation between Pleroma instances. +- User and conversation mutes can now auto-expire if `expires_in` parameter was given while adding the mute. ### Changed From 9c672ecbb5d4477cd16d2139a2cb66d3923ac5c8 Mon Sep 17 00:00:00 2001 From: Alex Gleason <alex@alexgleason.me> Date: Thu, 8 Oct 2020 20:01:48 -0500 Subject: [PATCH 13/74] Remote Timeline: add Streaming support --- CHANGELOG.md | 1 + docs/API/differences_in_mastoapi_responses.md | 6 ++++++ lib/pleroma/activity/ir/topics.ex | 13 ++++++++++++- lib/pleroma/web/streamer/streamer.ex | 9 +++++++++ test/activity/ir/topics_test.exs | 14 ++++++++++++++ test/integration/mastodon_websocket_test.exs | 1 + test/web/streamer/streamer_test.exs | 8 ++++++++ 7 files changed, 51 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fc1750d1..0eeffb72f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`) - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`) - Mix task option for force-unfollowing relays +- Ability to view remote timelines, with ex. `/api/v1/timelines/public?instance=lain.com` and streams `public:remote` and `public:remote:media` ### Changed diff --git a/docs/API/differences_in_mastoapi_responses.md b/docs/API/differences_in_mastoapi_responses.md index 38865dc68..bb1000b0b 100644 --- a/docs/API/differences_in_mastoapi_responses.md +++ b/docs/API/differences_in_mastoapi_responses.md @@ -9,9 +9,13 @@ Pleroma uses 128-bit ids as opposed to Mastodon's 64 bits. However just like Mas ## Timelines Adding the parameter `with_muted=true` to the timeline queries will also return activities by muted (not by blocked!) users. + Adding the parameter `exclude_visibilities` to the timeline queries will exclude the statuses with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`), e.g., `exclude_visibilities[]=direct&exclude_visibilities[]=private`. + Adding the parameter `reply_visibility` to the public and home timelines queries will filter replies. Possible values: without parameter (default) shows all replies, `following` - replies directed to you or users you follow, `self` - replies directed to you. +Adding the parameter `instance=lain.com` to the public timeline will show only statuses originating from `lain.com` (or any remote instance). + ## Statuses - `visibility`: has an additional possible value `list` @@ -249,6 +253,8 @@ Has these additional fields under the `pleroma` object: There is an additional `user:pleroma_chat` stream. Incoming chat messages will make the current chat be sent to this `user` stream. The `event` of an incoming chat message is `pleroma:chat_update`. The payload is the updated chat with the incoming chat message in the `last_message` field. +For viewing remote server timelines, there are `public:remote` and `public:remote:media` streams. Each of these accept a parameter like `?instance=lain.com`. + ## Not implemented Pleroma is generally compatible with the Mastodon 2.7.2 API, but some newer features and non-essential features are omitted. These features usually return an HTTP 200 status code, but with an empty response. While they may be added in the future, they are considered low priority. diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex index 9e65bedad..fe2e8cb5c 100644 --- a/lib/pleroma/activity/ir/topics.ex +++ b/lib/pleroma/activity/ir/topics.ex @@ -40,7 +40,8 @@ defp visibility_tags(object, activity) do end defp item_creation_tags(tags, object, %{data: %{"type" => "Create"}} = activity) do - tags ++ hashtags_to_topics(object) ++ attachment_topics(object, activity) + tags ++ + remote_topics(activity) ++ hashtags_to_topics(object) ++ attachment_topics(object, activity) end defp item_creation_tags(tags, _, _) do @@ -55,9 +56,19 @@ defp hashtags_to_topics(%{data: %{"tag" => tags}}) do defp hashtags_to_topics(_), do: [] + defp remote_topics(%{local: true}), do: [] + + defp remote_topics(%{actor: actor}) when is_binary(actor), + do: ["public:remote:" <> URI.parse(actor).host] + + defp remote_topics(_), do: [] + defp attachment_topics(%{data: %{"attachment" => []}}, _act), do: [] defp attachment_topics(_object, %{local: true}), do: ["public:media", "public:local:media"] + defp attachment_topics(_object, %{actor: actor}) when is_binary(actor), + do: ["public:media", "public:remote:media:" <> URI.parse(actor).host] + defp attachment_topics(_object, _act), do: ["public:media"] end diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex index 5475f18a6..d774f0dd9 100644 --- a/lib/pleroma/web/streamer/streamer.ex +++ b/lib/pleroma/web/streamer/streamer.ex @@ -57,6 +57,15 @@ def get_topic("hashtag", _user, _oauth_token, %{"tag" => tag} = _params) do {:ok, "hashtag:" <> tag} end + # Allow remote instance streams. + def get_topic("public:remote", _user, _oauth_token, %{"instance" => instance} = _params) do + {:ok, "public:remote:" <> instance} + end + + def get_topic("public:remote:media", _user, _oauth_token, %{"instance" => instance} = _params) do + {:ok, "public:remote:media:" <> instance} + end + # Expand user streams. def get_topic( stream, diff --git a/test/activity/ir/topics_test.exs b/test/activity/ir/topics_test.exs index 14a6e6b71..c8dcb28cc 100644 --- a/test/activity/ir/topics_test.exs +++ b/test/activity/ir/topics_test.exs @@ -93,6 +93,13 @@ test "only converts strings to hash tags", %{ refute Enum.member?(topics, "hashtag:2") end + + test "non-local action produces public:remote topic", %{activity: activity} do + activity = %{activity | local: false, actor: "https://lain.com/users/lain"} + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public:remote:lain.com") + end end describe "public visibility create events with attachments" do @@ -124,6 +131,13 @@ test "non-local doesn't produce public:local:media topics", %{activity: activity refute Enum.member?(topics, "public:local:media") end + + test "non-local action produces public:remote:media topic", %{activity: activity} do + activity = %{activity | local: false, actor: "https://lain.com/users/lain"} + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public:remote:media:lain.com") + end end describe "non-public visibility" do diff --git a/test/integration/mastodon_websocket_test.exs b/test/integration/mastodon_websocket_test.exs index 0f2e6cc2b..bb8e795b7 100644 --- a/test/integration/mastodon_websocket_test.exs +++ b/test/integration/mastodon_websocket_test.exs @@ -49,6 +49,7 @@ test "requires authentication and a valid token for protected streams" do test "allows public streams without authentication" do assert {:ok, _} = start_socket("?stream=public") assert {:ok, _} = start_socket("?stream=public:local") + assert {:ok, _} = start_socket("?stream=public:remote&instance=lain.com") assert {:ok, _} = start_socket("?stream=hashtag&tag=lain") end diff --git a/test/web/streamer/streamer_test.exs b/test/web/streamer/streamer_test.exs index 185724a9f..1495ed124 100644 --- a/test/web/streamer/streamer_test.exs +++ b/test/web/streamer/streamer_test.exs @@ -29,6 +29,14 @@ test "allows public" do assert {:ok, "public:local:media"} = Streamer.get_topic("public:local:media", nil, nil) end + test "allows instance streams" do + assert {:ok, "public:remote:lain.com"} = + Streamer.get_topic("public:remote", nil, nil, %{"instance" => "lain.com"}) + + assert {:ok, "public:remote:media:lain.com"} = + Streamer.get_topic("public:remote:media", nil, nil, %{"instance" => "lain.com"}) + end + test "allows hashtag streams" do assert {:ok, "hashtag:cofe"} = Streamer.get_topic("hashtag", nil, nil, %{"tag" => "cofe"}) end From 06934b820e739f8a5a598bb807235b621ca6e2ba Mon Sep 17 00:00:00 2001 From: Ali Riza Keles <ali@sifirbir.xyz> Date: Mon, 12 Oct 2020 23:20:10 +0100 Subject: [PATCH 14/74] Add ejabberd auth document --- docs/configuration/howto_ejabberd.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 docs/configuration/howto_ejabberd.md diff --git a/docs/configuration/howto_ejabberd.md b/docs/configuration/howto_ejabberd.md new file mode 100644 index 000000000..6940acbda --- /dev/null +++ b/docs/configuration/howto_ejabberd.md @@ -0,0 +1,9 @@ +# Configuring Ejabberd (XMPP Server) to use Pleroma for authentication + +If you want to give your Pleroma users an XMPP (chat) account, you can configure [Ejabberd](https://github.com/processone/ejabberd) to use your Pleroma server for user authentication, automatically giving every local user an XMPP account. + +In general, you just have to follow the configuration described at [https://docs.ejabberd.im/admin/configuration/authentication/#external-script](https://docs.ejabberd.im/admin/configuration/authentication/#external-script). Please read this section carefully. + +To get the external script please go to [https://github.com/alirizakeles/ejabberd-pleroma-auth](https://github.com/alirizakeles/ejabberd-pleroma-auth) and follow the steps described in README. + +After restarting your Ejabberd server, your users should now be able to connect with their Pleroma credentials. From 943f65c7399a03075cb6392ce89ae5403ab0c1a0 Mon Sep 17 00:00:00 2001 From: Ali Riza Keles <ali@sifirbir.xyz> Date: Tue, 13 Oct 2020 19:58:38 +0100 Subject: [PATCH 15/74] Include python script and description --- docs/configuration/howto_ejabberd.md | 129 ++++++++++++++++++++++++++- 1 file changed, 128 insertions(+), 1 deletion(-) diff --git a/docs/configuration/howto_ejabberd.md b/docs/configuration/howto_ejabberd.md index 6940acbda..520a0acbc 100644 --- a/docs/configuration/howto_ejabberd.md +++ b/docs/configuration/howto_ejabberd.md @@ -4,6 +4,133 @@ If you want to give your Pleroma users an XMPP (chat) account, you can configure In general, you just have to follow the configuration described at [https://docs.ejabberd.im/admin/configuration/authentication/#external-script](https://docs.ejabberd.im/admin/configuration/authentication/#external-script). Please read this section carefully. -To get the external script please go to [https://github.com/alirizakeles/ejabberd-pleroma-auth](https://github.com/alirizakeles/ejabberd-pleroma-auth) and follow the steps described in README. +Copy the script below to suitable path on your system and set owner and permissions. Also do not forget adjusting `PLEROMA_HOST` and `PLEROMA_PORT`, if necessary. + +```bash +cp pleroma_ejabberd_auth.py /etc/ejabberd/pleroma_ejabberd_auth.py +chown ejabberd /etc/ejabberd/pleroma_ejabberd_auth.py +chmod 700 /etc/ejabberd/pleroma_ejabberd_auth.py +``` + +Set external auth params in ejabberd.yaml file: + +```bash +auth_method: [external] +extauth_program: "python3 /etc/ejabberd/pleroma_ejabberd_auth.py" +extauth_instances: 3 +auth_use_cache: false +``` + +Restart / reload your ejabberd service. After restarting your Ejabberd server, your users should now be able to connect with their Pleroma credentials. + + +```python +import sys +import struct +import http.client +from base64 import b64encode +import logging + + +PLEROMA_HOST = "127.0.0.1" +PLEROMA_PORT = "4000" +AUTH_ENDPOINT = "/api/v1/accounts/verify_credentials" +USER_ENDPOINT = "/api/v1/accounts" +LOGFILE = "/var/log/ejabberd/pleroma_auth.log" + +logging.basicConfig(filename=LOGFILE, level=logging.INFO) + + +# Pleroma functions +def create_connection(): + return http.client.HTTPConnection(PLEROMA_HOST, PLEROMA_PORT) + + +def verify_credentials(user: str, password: str) -> bool: + user_pass_b64 = b64encode("{}:{}".format( + user, password).encode('utf-8')).decode("ascii") + params = {} + headers = { + "Authorization": "Basic {}".format(user_pass_b64) + } + + try: + conn = create_connection() + conn.request("GET", AUTH_ENDPOINT, params, headers) + + response = conn.getresponse() + if response.status == 200: + return True + + return False + except Exception as e: + logging.info("Can not connect: %s", str(e)) + return False + + +def does_user_exist(user: str) -> bool: + conn = create_connection() + conn.request("GET", "{}/{}".format(USER_ENDPOINT, user)) + + response = conn.getresponse() + if response.status == 200: + return True + + return False + + +def auth(username: str, server: str, password: str) -> bool: + return verify_credentials(username, password) + + +def isuser(username, server): + return does_user_exist(username) + + +def read(): + (pkt_size,) = struct.unpack('>H', bytes(sys.stdin.read(2), encoding='utf8')) + pkt = sys.stdin.read(pkt_size) + cmd = pkt.split(':')[0] + if cmd == 'auth': + username, server, password = pkt.split(':', 3)[1:] + write(auth(username, server, password)) + elif cmd == 'isuser': + username, server = pkt.split(':', 2)[1:] + write(isuser(username, server)) + elif cmd == 'setpass': + # u, s, p = pkt.split(':', 3)[1:] + write(False) + elif cmd == 'tryregister': + # u, s, p = pkt.split(':', 3)[1:] + write(False) + elif cmd == 'removeuser': + # u, s = pkt.split(':', 2)[1:] + write(False) + elif cmd == 'removeuser3': + # u, s, p = pkt.split(':', 3)[1:] + write(False) + else: + write(False) + + +def write(result): + if result: + sys.stdout.write('\x00\x02\x00\x01') + else: + sys.stdout.write('\x00\x02\x00\x00') + sys.stdout.flush() + + +if __name__ == "__main__": + logging.info("Starting pleroma ejabberd auth daemon...") + while True: + try: + read() + except Exception as e: + logging.info( + "Error while processing data from ejabberd %s", str(e)) + pass + +``` \ No newline at end of file From 3b5a7a6b14f4c09d1d371d6fcb49bece84d6c3e1 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me> Date: Fri, 16 Oct 2020 00:32:20 +0200 Subject: [PATCH 16/74] federation_status: New endpoint showing unreachable instances --- lib/pleroma/instances.ex | 1 + lib/pleroma/instances/instance.ex | 11 +++++ .../controllers/instances_controller.ex | 19 +++++++++ lib/pleroma/web/router.ex | 1 + .../controllers/instances_controller_test.exs | 40 +++++++++++++++++++ 5 files changed, 72 insertions(+) create mode 100644 lib/pleroma/web/pleroma_api/controllers/instances_controller.ex create mode 100644 test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex index 557e8decf..7315bd7cb 100644 --- a/lib/pleroma/instances.ex +++ b/lib/pleroma/instances.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Instances do defdelegate reachable?(url_or_host), to: @adapter defdelegate set_reachable(url_or_host), to: @adapter defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: @adapter + defdelegate get_consistently_unreachable(), to: @adapter def set_consistently_unreachable(url_or_host), do: set_unreachable(url_or_host, reachability_datetime_threshold()) diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index f0f601469..df471a39d 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -119,6 +119,17 @@ def set_unreachable(url_or_host, unreachable_since) when is_binary(url_or_host) def set_unreachable(_, _), do: {:error, nil} + def get_consistently_unreachable do + reachability_datetime_threshold = Instances.reachability_datetime_threshold() + + from(i in Instance, + where: ^reachability_datetime_threshold > i.unreachable_since, + order_by: i.unreachable_since, + select: {i.host, i.unreachable_since} + ) + |> Repo.all() + end + defp parse_datetime(datetime) when is_binary(datetime) do NaiveDateTime.from_iso8601(datetime) end diff --git a/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex b/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex new file mode 100644 index 000000000..bd95cb523 --- /dev/null +++ b/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex @@ -0,0 +1,19 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.InstancesController do + use Pleroma.Web, :controller + + alias Pleroma.Instances + + # defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaInstancesController + + def show(conn, _params) do + unreachable = + Instances.get_consistently_unreachable() + |> Enum.reduce(%{}, fn {host, date}, acc -> Map.put(acc, host, to_string(date)) end) + + json(conn, %{"unreachable" => unreachable}) + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index d2d939989..5f9a749e4 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -373,6 +373,7 @@ defmodule Pleroma.Web.Router do scope "/api/v1/pleroma", Pleroma.Web.PleromaAPI do pipe_through(:api) get("/accounts/:id/scrobbles", ScrobbleController, :index) + get("/federation_status", InstancesController, :show) end scope "/api/v1", Pleroma.Web.MastodonAPI do diff --git a/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs new file mode 100644 index 000000000..9ce901ce3 --- /dev/null +++ b/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs @@ -0,0 +1,40 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaApi.InstancesControllerTest do + use Pleroma.Web.ConnCase + + alias Pleroma.Instances + + setup_all do: clear_config([:instance, :federation_reachability_timeout_days], 1) + + setup do + constant = "http://consistently-unreachable.name/" + eventual = "http://eventually-unreachable.com/path" + + {:ok, %Pleroma.Instances.Instance{unreachable_since: constant_unreachable}} = + Instances.set_consistently_unreachable(constant) + + _eventual_unrechable = Instances.set_unreachable(eventual) + + %{constant_unreachable: constant_unreachable, constant: constant} + end + + test "GET /api/v1/pleroma/federation_status", %{ + conn: conn, + constant_unreachable: constant_unreachable, + constant: constant + } do + constant_host = URI.parse(constant).host + + assert conn + |> put_req_header("content-type", "application/json") + |> get("/api/v1/pleroma/federation_status") + |> json_response(200) == %{ + "unreachable" => %{constant_host => to_string(constant_unreachable)} + } + + # |> json_response_and_validate_schema(200) + end +end From aafdc975bdd38f74cdf5d3f8517d41c5dd76c56b Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me> Date: Fri, 16 Oct 2020 01:13:52 +0200 Subject: [PATCH 17/74] federation_status: Add ApiSpec --- .../operations/pleroma_instances_operation.ex | 40 +++++++++++++++++++ .../controllers/instances_controller.ex | 4 +- .../controllers/instances_controller_test.exs | 4 +- 3 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 lib/pleroma/web/api_spec/operations/pleroma_instances_operation.ex diff --git a/lib/pleroma/web/api_spec/operations/pleroma_instances_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_instances_operation.ex new file mode 100644 index 000000000..2c455b0df --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/pleroma_instances_operation.ex @@ -0,0 +1,40 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.PleromaInstancesOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def show_operation do + %Operation{ + tags: ["PleromaInstances"], + summary: "Instances federation status", + description: "Information about instances deemed unreachable by the server", + operationId: "PleromaInstances.show", + responses: %{ + 200 => Operation.response("PleromaInstances", "application/json", pleroma_instances()) + } + } + end + + def pleroma_instances do + %Schema{ + type: :object, + properties: %{ + unreachable: %Schema{ + type: :object, + properties: %{hostname: %Schema{type: :string, format: :"date-time"}} + } + }, + example: %{ + "unreachable" => %{"consistently-unreachable.name" => "2020-10-14 22:07:58.216473"} + } + } + end +end diff --git a/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex b/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex index bd95cb523..c577f1d1e 100644 --- a/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex @@ -7,7 +7,9 @@ defmodule Pleroma.Web.PleromaAPI.InstancesController do alias Pleroma.Instances - # defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaInstancesController + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaInstancesOperation def show(conn, _params) do unreachable = diff --git a/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs index 9ce901ce3..13491ed9c 100644 --- a/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs @@ -31,10 +31,8 @@ test "GET /api/v1/pleroma/federation_status", %{ assert conn |> put_req_header("content-type", "application/json") |> get("/api/v1/pleroma/federation_status") - |> json_response(200) == %{ + |> json_response_and_validate_schema(200) == %{ "unreachable" => %{constant_host => to_string(constant_unreachable)} } - - # |> json_response_and_validate_schema(200) end end From 2ca98f2d94e2976ae35998aecff27809d4b066cf Mon Sep 17 00:00:00 2001 From: Haelwenn <contact+git.pleroma.social@hacktivis.me> Date: Wed, 21 Oct 2020 19:40:37 +0000 Subject: [PATCH 18/74] Apply 1 suggestion(s) to 1 file(s) --- lib/pleroma/web/pleroma_api/controllers/instances_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex b/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex index c577f1d1e..9e97480df 100644 --- a/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex @@ -14,7 +14,7 @@ defmodule Pleroma.Web.PleromaAPI.InstancesController do def show(conn, _params) do unreachable = Instances.get_consistently_unreachable() - |> Enum.reduce(%{}, fn {host, date}, acc -> Map.put(acc, host, to_string(date)) end) + |> Map.new(fn {host, date} -> {host, to_string(date)} end) json(conn, %{"unreachable" => unreachable}) end From 241bd061fc60a5c90c172f46f3b4e576ba660aaf Mon Sep 17 00:00:00 2001 From: Alibek Omarov <a1ba.omarov@gmail.com> Date: Fri, 16 Oct 2020 18:28:27 +0000 Subject: [PATCH 19/74] ConversationView: add current user to conversations, according to Mastodon behaviour --- lib/pleroma/web/mastodon_api/views/conversation_view.ex | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex index a91994915..cf34933ab 100644 --- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -33,12 +33,10 @@ def render("participation.json", %{participation: participation, for: user}) do end activity = Activity.get_by_id_with_object(last_activity_id) - # Conversations return all users except the current user. - users = Enum.reject(participation.recipients, &(&1.id == user.id)) %{ id: participation.id |> to_string(), - accounts: render(AccountView, "index.json", users: users, for: user), + accounts: render(AccountView, "index.json", users: participation.recipients, for: user), unread: !participation.read, last_status: render(StatusView, "show.json", From 390a12d4c892e58e12546a78bc02dcc0e3a3484b Mon Sep 17 00:00:00 2001 From: Alibek Omarov <a1ba.omarov@gmail.com> Date: Sun, 18 Oct 2020 15:58:06 +0000 Subject: [PATCH 20/74] ConversationControllerTest: fix test --- .../mastodon_api/controllers/conversation_controller_test.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs index b23b22752..afc24027b 100644 --- a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs @@ -54,7 +54,8 @@ test "returns correct conversations", %{ ] = response account_ids = Enum.map(res_accounts, & &1["id"]) - assert length(res_accounts) == 2 + assert length(res_accounts) == 3 + assert user_one.id in account_ids assert user_two.id in account_ids assert user_three.id in account_ids assert is_binary(res_id) From 149589c842e677a082436db927834dd6f1b10cb5 Mon Sep 17 00:00:00 2001 From: Alibek Omarov <a1ba.omarov@gmail.com> Date: Sun, 18 Oct 2020 16:01:17 +0000 Subject: [PATCH 21/74] ConversationViewTest: fix test --- .../web/mastodon_api/views/conversation_view_test.exs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs index 2e8203c9b..bd58fb254 100644 --- a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs @@ -37,8 +37,10 @@ test "represents a Mastodon Conversation entity" do assert conversation.id == participation.id |> to_string() assert conversation.last_status.id == activity.id - assert [account] = conversation.accounts - assert account.id == other_user.id + account_ids = Enum.map(conversation.accounts, & &1["id"]) + assert length(conversation.accounts) == 2 + assert user.id in account_ids + assert other_user.id in account_ids assert conversation.last_status.pleroma.direct_conversation_id == participation.id end end From 630eb0f939013db721c78e9b33e4e8bdc8232834 Mon Sep 17 00:00:00 2001 From: Alibek Omarov <a1ba.omarov@gmail.com> Date: Sun, 18 Oct 2020 19:12:42 +0000 Subject: [PATCH 22/74] ConversationViewTest: fix test #2 --- test/pleroma/web/mastodon_api/views/conversation_view_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs index bd58fb254..81a471cb5 100644 --- a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs @@ -37,7 +37,7 @@ test "represents a Mastodon Conversation entity" do assert conversation.id == participation.id |> to_string() assert conversation.last_status.id == activity.id - account_ids = Enum.map(conversation.accounts, & &1["id"]) + account_ids = Enum.map(conversation.accounts, & &1.id) assert length(conversation.accounts) == 2 assert user.id in account_ids assert other_user.id in account_ids From 9b93eef71550eabf55b9728b6c8925a4dede222d Mon Sep 17 00:00:00 2001 From: Alibek Omarov <a1ba.omarov@gmail.com> Date: Fri, 30 Oct 2020 13:01:58 +0100 Subject: [PATCH 23/74] ConversationView: fix last_status.account being empty, fix current user being included in group conversations --- .../mastodon_api/views/conversation_view.ex | 12 +++++++-- .../conversation_controller_test.exs | 25 +++++++++++++++++-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex index cf34933ab..4636c00e3 100644 --- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -34,14 +34,22 @@ def render("participation.json", %{participation: participation, for: user}) do activity = Activity.get_by_id_with_object(last_activity_id) + # Conversations return all users except current user when current user is not only participant + users = if length(participation.recipients) > 1 do + Enum.reject(participation.recipients, &(&1.id == user.id)) + else + participation.recipients + end + %{ id: participation.id |> to_string(), - accounts: render(AccountView, "index.json", users: participation.recipients, for: user), + accounts: render(AccountView, "index.json", users: users, for: user), unread: !participation.read, last_status: render(StatusView, "show.json", activity: activity, - direct_conversation_id: participation.id + direct_conversation_id: participation.id, + for: user ) } end diff --git a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs index afc24027b..8d07cff3f 100644 --- a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs @@ -54,16 +54,37 @@ test "returns correct conversations", %{ ] = response account_ids = Enum.map(res_accounts, & &1["id"]) - assert length(res_accounts) == 3 - assert user_one.id in account_ids + assert length(res_accounts) == 2 + assert user_one.id not in account_ids assert user_two.id in account_ids assert user_three.id in account_ids assert is_binary(res_id) assert unread == false assert res_last_status["id"] == direct.id + assert res_last_status["account"]["id"] == user_one.id assert Participation.unread_count(user_one) == 0 end + test "special behaviour when conversation have only one user", %{ + user: user_one, + user_two: user_two, + conn: conn + } do + {:ok, direct} = create_direct_message(user_one, []) + + res_conn = get(conn, "/api/v1/conversations") + + assert response = json_response_and_validate_schema(res_conn, 200) + assert [ + %{ + "accounts" => res_accounts, + "last_status" => res_last_status + } + ] = response + assert length(res_accounts) == 1 + assert res_accounts[0]["id"] == user_one.id + end + test "observes limit params", %{ user: user_one, user_two: user_two, From 5591dc02486c30e4b80061706f7368d4b788b431 Mon Sep 17 00:00:00 2001 From: Alibek Omarov <a1ba.omarov@gmail.com> Date: Fri, 30 Oct 2020 13:07:01 +0100 Subject: [PATCH 24/74] Add entry in changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11820d313..c62d20868 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,8 @@ switched to a new configuration mechanism, however it was not officially removed - Allow sending chat messages to yourself. - Fix remote users with a whitespace name. - OStatus / static FE endpoints: fixed inaccessibility for anonymous users on non-federating instances, switched to handling per `:restrict_unauthenticated` setting. +- Mastodon API: Current user is now included in conversation if it's the only participant +- Mastodon API: Fixed last_status.account being not filled with account data ## Unreleased (Patch) From 0552a08dfd4daeca69abca0274bbd6db018e5edb Mon Sep 17 00:00:00 2001 From: Alibek Omarov <a1ba.omarov@gmail.com> Date: Fri, 30 Oct 2020 13:10:19 +0100 Subject: [PATCH 25/74] ConversationControllerTest: fix test, fix formatting --- .../controllers/conversation_controller_test.exs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs index 8d07cff3f..291b6b295 100644 --- a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs @@ -75,14 +75,17 @@ test "special behaviour when conversation have only one user", %{ res_conn = get(conn, "/api/v1/conversations") assert response = json_response_and_validate_schema(res_conn, 200) + assert [ %{ "accounts" => res_accounts, "last_status" => res_last_status } ] = response + + account_ids = Enum.map(res_accounts, & &1["id"]) assert length(res_accounts) == 1 - assert res_accounts[0]["id"] == user_one.id + assert user_one.id in account_ids end test "observes limit params", %{ From d63ec02f31e5ee7bb278c4247a83900aceb9193a Mon Sep 17 00:00:00 2001 From: Alibek Omarov <a1ba.omarov@gmail.com> Date: Fri, 30 Oct 2020 13:25:13 +0100 Subject: [PATCH 26/74] ConversationView: fix formatting --- .../web/mastodon_api/views/conversation_view.ex | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex index 4636c00e3..545778165 100644 --- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -35,11 +35,12 @@ def render("participation.json", %{participation: participation, for: user}) do activity = Activity.get_by_id_with_object(last_activity_id) # Conversations return all users except current user when current user is not only participant - users = if length(participation.recipients) > 1 do - Enum.reject(participation.recipients, &(&1.id == user.id)) - else - participation.recipients - end + users = + if length(participation.recipients) > 1 do + Enum.reject(participation.recipients, &(&1.id == user.id)) + else + participation.recipients + end %{ id: participation.id |> to_string(), From 1042c30fa53e838f3acae2c176f47997fa425755 Mon Sep 17 00:00:00 2001 From: Alibek Omarov <a1ba.omarov@gmail.com> Date: Fri, 30 Oct 2020 13:37:15 +0100 Subject: [PATCH 27/74] ConversationViewTest: fix test --- .../pleroma/web/mastodon_api/views/conversation_view_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs index 81a471cb5..cd02158f9 100644 --- a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs @@ -36,10 +36,10 @@ test "represents a Mastodon Conversation entity" do assert conversation.id == participation.id |> to_string() assert conversation.last_status.id == activity.id + assert conversation.last_status.account.id == user.id account_ids = Enum.map(conversation.accounts, & &1.id) - assert length(conversation.accounts) == 2 - assert user.id in account_ids + assert length(conversation.accounts) == 1 assert other_user.id in account_ids assert conversation.last_status.pleroma.direct_conversation_id == participation.id end From 8f00d90f9199e384fb1befb677c1c0595a0c854c Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis <vaartis@cock.li> Date: Sun, 1 Nov 2020 12:05:39 +0300 Subject: [PATCH 28/74] Use Pleroma.HTTP instead of Tesla Closes #2275 As discovered in the issue, captcha used Tesla.get instead of Pleroma.HTTP. I've also grep'ed the repo and changed the other place where this was used. --- lib/pleroma/captcha/kocaptcha.ex | 2 +- lib/pleroma/emoji/pack.ex | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/captcha/kocaptcha.ex b/lib/pleroma/captcha/kocaptcha.ex index 337506647..201b55ab4 100644 --- a/lib/pleroma/captcha/kocaptcha.ex +++ b/lib/pleroma/captcha/kocaptcha.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Captcha.Kocaptcha do def new do endpoint = Pleroma.Config.get!([__MODULE__, :endpoint]) - case Tesla.get(endpoint <> "/new") do + case Pleroma.HTTP.get(endpoint <> "/new") do {:error, _} -> %{error: :kocaptcha_service_unavailable} diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index 0670f29f1..ca58e5432 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -594,7 +594,7 @@ defp fetch_pack_info(remote_pack, uri, name) do end defp download_archive(url, sha) do - with {:ok, %{body: archive}} <- Tesla.get(url) do + with {:ok, %{body: archive}} <- Pleroma.HTTP.get(url) do if Base.decode16!(sha) == :crypto.hash(:sha256, archive) do {:ok, archive} else @@ -617,7 +617,7 @@ defp fallback_sha_changed?(pack, data) do end defp update_sha_and_save_metadata(pack, data) do - with {:ok, %{body: zip}} <- Tesla.get(data[:"fallback-src"]), + with {:ok, %{body: zip}} <- Pleroma.HTTP.get(data[:"fallback-src"]), :ok <- validate_has_all_files(pack, zip) do fallback_sha = :sha256 |> :crypto.hash(zip) |> Base.encode16() From 4caad4e9101c34debfa90d2e89850d4125a471b3 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me> Date: Mon, 2 Nov 2020 05:43:06 +0100 Subject: [PATCH 29/74] =?UTF-8?q?side=5Feffects:=20Don=E2=80=99t=20increas?= =?UTF-8?q?e=5Freplies=5Fcount=20when=20it=E2=80=99s=20an=20Answer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/pleroma/web/activity_pub/side_effects.ex | 2 +- .../web/activity_pub/transmogrifier/answer_handling_test.exs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 0fff5faf2..9b1171d07 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -187,7 +187,7 @@ def handle(%{data: %{"type" => "Create"}} = activity, meta) do {:ok, notifications} = Notification.create_notifications(activity, do_send: false) {:ok, _user} = ActivityPub.increase_note_count_if_public(user, object) - if in_reply_to = object.data["inReplyTo"] do + if in_reply_to = object.data["inReplyTo"] && object.data["type"] != "Answer" do Object.increase_replies_count(in_reply_to) end diff --git a/test/pleroma/web/activity_pub/transmogrifier/answer_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/answer_handling_test.exs index 0f6605c3f..e7d85a2c5 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/answer_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/answer_handling_test.exs @@ -27,6 +27,7 @@ test "incoming, rewrites Note to Answer and increments vote counters" do }) object = Object.normalize(activity) + assert object.data["repliesCount"] == nil data = File.read!("test/fixtures/mastodon-vote.json") @@ -41,7 +42,7 @@ test "incoming, rewrites Note to Answer and increments vote counters" do assert answer_object.data["inReplyTo"] == object.data["id"] new_object = Object.get_by_ap_id(object.data["id"]) - assert new_object.data["replies_count"] == object.data["replies_count"] + assert new_object.data["repliesCount"] == nil assert Enum.any?( new_object.data["oneOf"], From be52819a112abb66032a56d613eed0233995eef4 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn <egor@kislitsyn.com> Date: Mon, 2 Nov 2020 17:51:54 +0400 Subject: [PATCH 30/74] Hide chats from muted users --- .../controllers/chat_controller.ex | 27 ++++++++----------- .../controllers/chat_controller_test.exs | 22 +++++++++++++++ 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex index 2c4d3f135..8fc70c15a 100644 --- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex @@ -15,7 +15,6 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do alias Pleroma.User alias Pleroma.Web.CommonAPI alias Pleroma.Web.PleromaAPI.Chat.MessageReferenceView - alias Pleroma.Web.PleromaAPI.ChatView alias Pleroma.Web.Plugs.OAuthScopesPlug import Ecto.Query @@ -121,9 +120,7 @@ def mark_as_read( ) do with {:ok, chat} <- Chat.get_by_user_and_id(user, id), {_n, _} <- MessageReference.set_all_seen_for_chat(chat, last_read_id) do - conn - |> put_view(ChatView) - |> render("show.json", chat: chat) + render(conn, "show.json", chat: chat) end end @@ -142,32 +139,30 @@ def messages(%{assigns: %{user: user}} = conn, %{id: id} = params) do end def index(%{assigns: %{user: %{id: user_id} = user}} = conn, _params) do - blocked_ap_ids = User.blocked_users_ap_ids(user) + exclude_users = + user + |> User.blocked_users_ap_ids() + |> Enum.concat(User.muted_users_ap_ids(user)) chats = - Chat.for_user_query(user_id) - |> where([c], c.recipient not in ^blocked_ap_ids) + user_id + |> Chat.for_user_query() + |> where([c], c.recipient not in ^exclude_users) |> Repo.all() - conn - |> put_view(ChatView) - |> render("index.json", chats: chats) + render(conn, "index.json", chats: chats) end def create(%{assigns: %{user: user}} = conn, %{id: id}) do with %User{ap_id: recipient} <- User.get_cached_by_id(id), {:ok, %Chat{} = chat} <- Chat.get_or_create(user.id, recipient) do - conn - |> put_view(ChatView) - |> render("show.json", chat: chat) + render(conn, "show.json", chat: chat) end end def show(%{assigns: %{user: user}} = conn, %{id: id}) do with {:ok, chat} <- Chat.get_by_user_and_id(user, id) do - conn - |> put_view(ChatView) - |> render("show.json", chat: chat) + render(conn, "show.json", chat: chat) end end diff --git a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs index fa6b9db65..b0498df2b 100644 --- a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs @@ -343,6 +343,28 @@ test "it does not return chats with users you blocked", %{conn: conn, user: user assert length(result) == 0 end + test "it does not return chats with users you muted", %{conn: conn, user: user} do + recipient = insert(:user) + + {:ok, _} = Chat.get_or_create(user.id, recipient.ap_id) + + result = + conn + |> get("/api/v1/pleroma/chats") + |> json_response_and_validate_schema(200) + + assert length(result) == 1 + + User.mute(user, recipient) + + result = + conn + |> get("/api/v1/pleroma/chats") + |> json_response_and_validate_schema(200) + + assert length(result) == 0 + end + test "it returns all chats", %{conn: conn, user: user} do Enum.each(1..30, fn _ -> recipient = insert(:user) From 7efc074eadae9b3d6d351e769ead0661f1f4c89c Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@FreeBSD.org> Date: Mon, 2 Nov 2020 12:19:44 -0600 Subject: [PATCH 31/74] Permit fetching individual reports with notes preloaded --- lib/pleroma/activity.ex | 13 +++++++++++++ .../web/admin_api/controllers/report_controller.ex | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 17af04257..553834da0 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -14,6 +14,7 @@ defmodule Pleroma.Activity do alias Pleroma.ReportNote alias Pleroma.ThreadMute alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub import Ecto.Changeset import Ecto.Query @@ -153,6 +154,18 @@ def get_bookmark(%Activity{} = activity, %User{} = user) do def get_bookmark(_, _), do: nil + def get_report(activity_id) do + opts = %{ + type: "Flag", + skip_preload: true, + preload_report_notes: true + } + + ActivityPub.fetch_activities_query([], opts) + |> where(id: ^activity_id) + |> Repo.one() + end + def change(struct, params \\ %{}) do struct |> cast(params, [:data, :recipients]) diff --git a/lib/pleroma/web/admin_api/controllers/report_controller.ex b/lib/pleroma/web/admin_api/controllers/report_controller.ex index 86da93893..6a0e56f5f 100644 --- a/lib/pleroma/web/admin_api/controllers/report_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/report_controller.ex @@ -38,7 +38,7 @@ def index(conn, params) do end def show(conn, %{id: id}) do - with %Activity{} = report <- Activity.get_by_id(id) do + with %Activity{} = report <- Activity.get_report(id) do render(conn, "show.json", Report.extract_report_info(report)) else _ -> {:error, :not_found} From 53dd048590b93da67f9d4abac8cc111424c4d5c0 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@FreeBSD.org> Date: Mon, 2 Nov 2020 15:49:07 -0600 Subject: [PATCH 32/74] Test the note is returned when fetching a single report --- .../web/admin_api/controllers/report_controller_test.exs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/pleroma/web/admin_api/controllers/report_controller_test.exs b/test/pleroma/web/admin_api/controllers/report_controller_test.exs index fa746d6ea..958e1d3ab 100644 --- a/test/pleroma/web/admin_api/controllers/report_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/report_controller_test.exs @@ -37,12 +37,21 @@ test "returns report by its id", %{conn: conn} do status_ids: [activity.id] }) + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/reports/#{report_id}/notes", %{ + content: "this is an admin note" + }) + response = conn |> get("/api/pleroma/admin/reports/#{report_id}") |> json_response_and_validate_schema(:ok) assert response["id"] == report_id + + [notes] = response["notes"] + assert notes["content"] == "this is an admin note" end test "returns 404 when report id is invalid", %{conn: conn} do From 2f2281fdf1bd3fbd5d82bb437ce3d43ff9043862 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@FreeBSD.org> Date: Mon, 2 Nov 2020 17:09:56 -0600 Subject: [PATCH 33/74] Ensure URLs for git repos end in .git for older git clients like on CentOS 7 --- mix.exs | 4 ++-- mix.lock | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mix.exs b/mix.exs index e0da696ce..0691902a6 100644 --- a/mix.exs +++ b/mix.exs @@ -134,7 +134,7 @@ defp deps do {:cachex, "~> 3.2"}, {:poison, "~> 3.0", override: true}, {:tesla, - git: "https://github.com/teamon/tesla/", + git: "https://github.com/teamon/tesla.git", ref: "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30", override: true}, {:castore, "~> 0.1"}, @@ -196,7 +196,7 @@ defp deps do ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"}, {:restarter, path: "./restarter"}, {:majic, - git: "https://git.pleroma.social/pleroma/elixir-libraries/majic", branch: "develop"}, + git: "https://git.pleroma.social/pleroma/elixir-libraries/majic.git", branch: "develop"}, {:open_api_spex, git: "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git", ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"}, diff --git a/mix.lock b/mix.lock index 07238f550..e5d9bc693 100644 --- a/mix.lock +++ b/mix.lock @@ -66,7 +66,7 @@ "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"}, "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"}, "linkify": {:hex, :linkify, "0.2.0", "2518bbbea21d2caa9d372424e1ad845b640c6630e2d016f1bd1f518f9ebcca28", [:mix], [], "hexpm", "b8ca8a68b79e30b7938d6c996085f3db14939f29538a59ca5101988bb7f917f6"}, - "majic": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/majic", "4c692e544b28d1f5e543fb8a44be090f8cd96f80", [branch: "develop"]}, + "majic": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/majic.git", "4c692e544b28d1f5e543fb8a44be090f8cd96f80", [branch: "develop"]}, "makeup": {:hex, :makeup, "1.0.3", "e339e2f766d12e7260e6672dd4047405963c5ec99661abdc432e6ec67d29ef95", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "2e9b4996d11832947731f7608fed7ad2f9443011b3b479ae288011265cdd3dad"}, "makeup_elixir": {:hex, :makeup_elixir, "0.14.1", "4f0e96847c63c17841d42c08107405a005a2680eb9c7ccadfd757bd31dabccfb", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f2438b1a80eaec9ede832b5c41cd4f373b38fd7aa33e3b22d9db79e640cbde11"}, "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm", "d34f013c156db51ad57cc556891b9720e6a1c1df5fe2e15af999c84d6cebeb1a"}, @@ -115,7 +115,7 @@ "swoosh": {:hex, :swoosh, "1.0.6", "6765e334c67dacabe721f0d701c7e5a6f06e4595c90df6f91e73ebd54d555833", [:mix], [{:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "7c50ef78e4acfd1cbd4907dc1fa87b5540675a6be9dc979d04890f49d7ec1830"}, "syslog": {:hex, :syslog, "1.1.0", "6419a232bea84f07b56dc575225007ffe34d9fdc91abe6f1b2f254fd71d8efc2", [:rebar3], [], "hexpm", "4c6a41373c7e20587be33ef841d3de6f3beba08519809329ecc4d27b15b659e1"}, "telemetry": {:hex, :telemetry, "0.4.2", "2808c992455e08d6177322f14d3bdb6b625fbcfd233a73505870d8738a2f4599", [:rebar3], [], "hexpm", "2d1419bd9dda6a206d7b5852179511722e2b18812310d304620c7bd92a13fcef"}, - "tesla": {:git, "https://github.com/teamon/tesla/", "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30", [ref: "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30"]}, + "tesla": {:git, "https://github.com/teamon/tesla.git", "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30", [ref: "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30"]}, "timex": {:hex, :timex, "3.6.2", "845cdeb6119e2fef10751c0b247b6c59d86d78554c83f78db612e3290f819bc2", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "26030b46199d02a590be61c2394b37ea25a3664c02fafbeca0b24c972025d47a"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"}, "tzdata": {:hex, :tzdata, "1.0.4", "a3baa4709ea8dba552dca165af6ae97c624a2d6ac14bd265165eaa8e8af94af6", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "b02637db3df1fd66dd2d3c4f194a81633d0e4b44308d36c1b2fdfd1e4e6f169b"}, From 179936609f4fbf51575fabd7af11cf14ba570c0c Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me> Date: Mon, 2 Nov 2020 06:11:14 +0100 Subject: [PATCH 34/74] favicon: Update to pleroma logo, provided by @shpuld Closes: https://git.pleroma.social/pleroma/pleroma/-/issues/2270 --- priv/static/favicon.png | Bin 1603 -> 1583 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/priv/static/favicon.png b/priv/static/favicon.png index a96d5d25225abd92a95b35c58b301463752cf335..098040a00d025387fc376f534fb6cb5aef842761 100644 GIT binary patch literal 1583 zcmds2`#TeQ9R6+=vrN&1qgYOg43(tl%qEv6*=#G4TS}C)b6ldcu}eZO;gEK6O_v8n ziaD&Ba>>eNbsmbbxnFXZT*5f(=l+B9d7k(4KF|App67kupFTH9E{;gVF9-ktl5ot9 zEN9jaz!l}Y4tusqPOwm00u?SF2K;<H02_u0c6e%}_hb$+{$x0+ahgscPKx3WSvkd+ z9!U1!qG1MXV|>g(lWdQw4<~~(U8_<H{Lr<$D-`Z&J5tp#E`D2Z``cKELLF!8Kt-{` z^xP|cKp*=0nqX1TGo4GH>RahGc^eu%ojoUup2~}eoBjMQ%%>|MWF>KNyn#-f%{7#Y z#wvSD#nfgyQ9Z`A`jkp^d1PKer-25=`LPbmGU0TqVQU>(L!)pbI6O8KZ{zt)?~$=F zNeN^nXk$&J*0%%BR;!^#Z<Xz;u)D>$t49{PV=O?3cqEP1JV>%Of~4wTLqD%xsgKzg zbhIGOPi#SvC<u#czDd?z&}#UTbpvtUhsz=w2Zu)vUV$sC>x^&zE#hLENK3UGQxmdK zUpP*AlK7>@Sw;oIMaGi2dcHxr>b5&&BDeebhF#`(d$!!**(_OAMKo@-Y|X<;zl3qF zsf_K}{KpN?Agb=PqTa|Y+E}s)BWhWz<+H^<&O8(lmdkp_m%mdS`R8&_JWjaQlTsXl zGN@@yjLNw-T_RcW&Q0}Mp7AHKQD8#3Wxkc|m45Hgxt2YtP*(3pnYeWFO=S5%`JV{D zv1FN@NJ^nxc!78TUQdNhsO-Ukd^^DwIE%*Y3xfftL2EWxTF`WH{C~)8p0-lGBe3=r zv}&EaU7KfZzhsD>Z<TO;=(K~pVv9j@vn)i3x~@@wmJ4t<?-G^*czHyT=Q@&Fv!Q>l z^N*XIjG3Vc#?AM~4MI3ZFtj1j{7s6P_p4c1OV{YKsU@rF+4e_bbHmC_rDmulx`l}7 z!n9tO)T;0I`NmJRJpfI$ZHO0Dt>^dnEDt=r{de`52q(`uy{BR31ruU26i^v!9RYXn zPDL}ZQujqXR%=1&`nr<|eD?{>zR>_c=75+Rt?JsD^KMECq~gm<9%)OVo#!(evv3bm zOP2ENtqG;Fl(+hzQbjfV*I_34WqS6qo<_d`dS`VGH+x}7NW3ISas(6H)e4uvShG$G zW|SJV%v(*zrYX(T@g<}EcyyZ$Rv!{5{e7Ri?yAu2>R_kfh%`C6*6k_=n(9ySZ}`e> zq(OAOO7?Dn6rwX5snFh!?Fe%<7Q*kEvjO<Ym_3ZkjH2Mp5{q0#`yv&~0*m~JFr4e< z#D{Q}y&H<I4-)*&VC(^gU(m4yLPB-<89{xUZ%<h{ep=7wdtsG8!mwLwE->Q7(j7%! z#pOT*!i&2&qFumbZ&B-3WgJeze_F%-H|QRpjs_6T*g71^h#Ij8hI@DHu|J%3G9Y7F zA2`{bF*ypJSlw^agpkz5m$ae2?b;6oaMoWRPBm=?k6R7r?LkINrAI7q4|gasgXZt7 zXx|TeU2N_Yr3MV{m1OP!r*d4!5`m#|Fi!<iHkL&8f#C>&KSxl$HAE>Ov+jQs_e<!3 z<65Y=;IAsc;F2l6G7ZvRO+#-k<n=M9LwjC-x_hT!wn!8(;>1v0L}GT3$H`d!%s}lX z8;Lh0IkX3dCXLsIh?(n&o!`~79Nu3hEU5P9#aSj1a&5L@)&$SH+XU<!*j&~cb73&W z*ip8^=kk=pkWqHegU@#}k=xoCt~M8fg|CEKi^wEYSxZwj%{NPuIFYlW=z!=L9_qF% zYnifN$eoCtCPd8y^`=uVuJu&WeCKov!#|7)*y{?y^p5PX3_L{s0RX|?#je!Wm-8RQ CN89cI literal 1603 zcmV-J2E6%+P)<h;3K|Lk000e1NJLTq001Tc001%w1ONa4nsZ^O000IGNkl<Zcmb`! z3(ydP0SEB^*Z1*Nx}>M3rg3eNl1HVk(c>Z|vaX`mV@f0+I+dn}9<xLbX_D$A^>wE; ztwfg|tVJYaxe1%cj7~P@+Og5aZ|CblDYV<~mm?u&nW0_&H=?q_loJD-?8mk^AxBCS zX>OZSl0-CD<Sxq$vRE|MDc_B_$U`1?g;FDuM4aO(g<4zXr=p?-=H%OmLK~c-h9VE^ zZ@3moBHCMGrI{%whH05^A}V=Eg+$CW+i^x3m~x_qhN88_i5ROxzKIx_h$=3(T3^w_ zyYA3gVakcR$~4u-#(X=`UU%yhn`(~d+^@Md)+tCi(aBQl6pLchot`g=y4g>Z8sTH> z6EQ{;T}=0?X)5FZqS(c1S?HMTA&RY3IpxHadU)5F3KeN=oQXDiUE3T)lsU#@&a*_* z>>*0LW~%<K)?F1HJ*HwJ>buP;##*bo%{hp;LrW)FYqGlelIUoSxsG+I*pMWmjY*ER zz-EKImV=0)R%qi9r7GmBL~S>UX-@K#sHBcghB?=r{%M)Y`5MvDSeN8bL|x~bVtXRC znXRo}%KQo~73gf7!RDK+UdoB%Ra4us>f}q}Y*F0+)eN)JG|lXAy<#!UUo7#F5!M*w zT94~xRFa69Hk;~M?_>`#%3Pxp(Zx3XEHcDRs;Xm>yF4$dxlyYm5e070OV=b31;*K; zb`mkbJ*GQ3<-}nBu}&|;Q%=kiRc!D?_7Y`wIyw=9ZE<40;6z&u%~yz}#+vF@Rg*-# zE4q5!2P!0@$T&OA(jpO^lzP$}%_3@Bs(ZddbhFjninVd7C3g9dKYPI@)l^nXXYIYA zt==goPP0;3z92T4=jiMu8lg-yRBD>FF4MxN`Z(RC)~Kb#W{q5%B;qs^P0BuEvMSk2 z^zgO|^-RPAK5?Ia`I9r8sfSCvAZqF0vP2B`l1e!o(c3b$y)IfAZGx+m_*_(UzpVy| z-&>+pA_{XL(I)$eZpJtcGu)=C4&KuZTXa`5NklDFO~m!OC^1F%B%->BZpwb5w#)VK zoKcP!*IH;i8i-m>P)T)f>z;@jN)>ufQ*)zGYm2PT0YnQkZL(Ec7pmt0zxJ^Ox;x82 zy=~VkNkn5a)b>}gJhphzts3P3qOJbcSmiv;6Y-W+c*RB=T#~&+ffXLM!zkCqP@_Dk zK@K2#d(mnG%+c2zW8LBd^vprT(eAZTmn7m`TiuZZh;vQ$sEZw?pWj%m#GAG$%Qq7Z zy{uHs(KY2nGru(7EOp%O6<Y+Kc{T?SHB3yzWX-dO_`XJ#8)l#qQx%C)eXSCAdeSnn z8=vO@;%svhG2Ha*BSu<jwG-60$didU)^ah&)8>jV>{2NaH4RG=(cBc(6u8pF>>(Ps zOcT_zPFr*k_01Ngwwf*$+ijQcC8DxVd}zM&#d^PRvuCo8*l4R>{w<oh*mlE2Q}a|% z=2PqS^R&-!d?FU9pn|wr1yQIl`-rLvL=$6OsGp&BnjmW1VYM}0_LisJj!zRY+sWd0 zHffi`5j7oWuEn++pNN-qH3Dl5aRnA8VyS8VX1*bM<?uv-1(tZ<@I=fM({Pb{mHOO> zL@d_RZDOYW`8rW#d?Jo<ohN+cnncV~!$$9E;t$U8j=qVQXqKx)pB#!9p=TnhTHqcp zxhfGe#ZFXmyQV(YGZDAA!NWN;QS5;v5f!bl!VaSo@kcGOOH5YRZdWJbZqvP*LlKqz z%uU8yrk$8(p(%EY{yLk631Xm{{_aiD*frReLlH$<J4+{<R1w7%c+dpBP0`Ip)Dp!W z@k2vgZJ=)6wm63;N^}<O#SJbs+M_yKjGrWlIL|%0i2MA*PCv-ui2l}?ZoOMI61SLd zIo{VPdx;XP^SlrB%i)RfCO9Yii3a%^QO{91GNO9^UqnX_Iw=to5>cc;E6vq6Us;ZX zXk&3ADx2p5QDlvdMrvb_pXW%3J|-q&j0ILEV!A?OG}Y3ee0SnZj~Of)dd5(9DG@83 zp`8mXbb5}2I8}Xh{nm?Oi5c2C%6w0G!=xO>e*siY3nVrpr!xQm002ovPDHLkV1irG B2owMS From c37118e6f26f0305d540047e4ccb8d594d2c0e6b Mon Sep 17 00:00:00 2001 From: lain <lain@soykaf.club> Date: Tue, 3 Nov 2020 13:56:12 +0100 Subject: [PATCH 35/74] Conversations: A few refactors --- .../web/mastodon_api/views/conversation_view.ex | 3 ++- .../controllers/conversation_controller_test.exs | 12 ++++-------- .../mastodon_api/views/conversation_view_test.exs | 6 +++--- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex index 545778165..82fcff062 100644 --- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -34,7 +34,8 @@ def render("participation.json", %{participation: participation, for: user}) do activity = Activity.get_by_id_with_object(last_activity_id) - # Conversations return all users except current user when current user is not only participant + # Conversations return all users except the current user, + # except when the current user is the only participant users = if length(participation.recipients) > 1 do Enum.reject(participation.recipients, &(&1.id == user.id)) diff --git a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs index 291b6b295..c67e584dd 100644 --- a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs @@ -65,12 +65,11 @@ test "returns correct conversations", %{ assert Participation.unread_count(user_one) == 0 end - test "special behaviour when conversation have only one user", %{ + test "includes the user if the user is the only participant", %{ user: user_one, - user_two: user_two, conn: conn } do - {:ok, direct} = create_direct_message(user_one, []) + {:ok, _direct} = create_direct_message(user_one, []) res_conn = get(conn, "/api/v1/conversations") @@ -78,14 +77,11 @@ test "special behaviour when conversation have only one user", %{ assert [ %{ - "accounts" => res_accounts, - "last_status" => res_last_status + "accounts" => [account] } ] = response - account_ids = Enum.map(res_accounts, & &1["id"]) - assert length(res_accounts) == 1 - assert user_one.id in account_ids + assert user_one.id == account["id"] end test "observes limit params", %{ diff --git a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs index cd02158f9..20c10ba3d 100644 --- a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs @@ -38,9 +38,9 @@ test "represents a Mastodon Conversation entity" do assert conversation.last_status.id == activity.id assert conversation.last_status.account.id == user.id - account_ids = Enum.map(conversation.accounts, & &1.id) - assert length(conversation.accounts) == 1 - assert other_user.id in account_ids + assert [account] = conversation.accounts + assert account.id == other_user.id + assert conversation.last_status.pleroma.direct_conversation_id == participation.id end end From 1cfc3278c086c9eaa7b2d1bd170e82c8b2aebd78 Mon Sep 17 00:00:00 2001 From: lain <lain@soykaf.club> Date: Wed, 4 Nov 2020 10:14:00 +0100 Subject: [PATCH 36/74] Poll View: Always return `voters_count`. --- lib/pleroma/web/mastodon_api/views/poll_view.ex | 2 +- test/pleroma/web/mastodon_api/views/poll_view_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/poll_view.ex b/lib/pleroma/web/mastodon_api/views/poll_view.ex index 1208dc9a0..4101f21d0 100644 --- a/lib/pleroma/web/mastodon_api/views/poll_view.ex +++ b/lib/pleroma/web/mastodon_api/views/poll_view.ex @@ -19,7 +19,7 @@ def render("show.json", %{object: object, multiple: multiple, options: options} expired: expired, multiple: multiple, votes_count: votes_count, - voters_count: (multiple || nil) && voters_count(object), + voters_count: voters_count(object), options: options, voted: voted?(params), emojis: Pleroma.Web.MastodonAPI.StatusView.build_emojis(object.data["emoji"]) diff --git a/test/pleroma/web/mastodon_api/views/poll_view_test.exs b/test/pleroma/web/mastodon_api/views/poll_view_test.exs index b7e2f17ef..c655ca438 100644 --- a/test/pleroma/web/mastodon_api/views/poll_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/poll_view_test.exs @@ -44,7 +44,7 @@ test "renders a poll" do ], voted: false, votes_count: 0, - voters_count: nil + voters_count: 0 } result = PollView.render("show.json", %{object: object}) From f09bb814a96c71f24fcf6e403a25e90be9cc684e Mon Sep 17 00:00:00 2001 From: lain <lain@soykaf.club> Date: Wed, 4 Nov 2020 10:14:48 +0100 Subject: [PATCH 37/74] Changelog: Add info about poll view changes --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ee17d239..8c5a9f9dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Users with the `discoverable` field set to false will not show up in searches. - Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option). - Introduced optional dependencies on `ffmpeg`, `ImageMagick`, `exiftool` software packages. Please refer to `docs/installation/optional/media_graphics_packages.md`. +- Polls now always return a `voters_count`, even if they are single-choice <details> <summary>API Changes</summary> From 92d252f364ed421f2afcdd135507ced3554eb3f0 Mon Sep 17 00:00:00 2001 From: lain <lain@soykaf.club> Date: Wed, 4 Nov 2020 10:20:09 +0100 Subject: [PATCH 38/74] Poll Schema: Update and fix. --- lib/pleroma/web/api_spec/schemas/poll.ex | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/api_spec/schemas/poll.ex b/lib/pleroma/web/api_spec/schemas/poll.ex index c62096db0..0dfa60b97 100644 --- a/lib/pleroma/web/api_spec/schemas/poll.ex +++ b/lib/pleroma/web/api_spec/schemas/poll.ex @@ -28,8 +28,11 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Poll do }, votes_count: %Schema{ type: :integer, - nullable: true, - description: "How many votes have been received. Number, or null if `multiple` is false." + description: "How many votes have been received. Number." + }, + voters_count: %Schema{ + type: :integer, + description: "How many unique accounts have voted. Number." }, voted: %Schema{ type: :boolean, @@ -61,7 +64,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Poll do expired: true, multiple: false, votes_count: 10, - voters_count: nil, + voters_count: 10, voted: true, own_votes: [ 1 From ca95cbe0b48b6c64e6e33addf79e4d212d5f9872 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn <egor@kislitsyn.com> Date: Wed, 4 Nov 2020 16:40:12 +0400 Subject: [PATCH 39/74] Add `with_muted` param to ChatController.index/2 --- docs/API/chats.md | 4 ++++ lib/pleroma/web/api_spec/operations/chat_operation.ex | 6 +++++- lib/pleroma/web/api_spec/operations/timeline_operation.ex | 2 +- lib/pleroma/web/pleroma_api/controllers/chat_controller.ex | 7 +++---- .../web/pleroma_api/controllers/chat_controller_test.exs | 7 +++++++ 5 files changed, 20 insertions(+), 6 deletions(-) diff --git a/docs/API/chats.md b/docs/API/chats.md index 9857aac67..f50144c86 100644 --- a/docs/API/chats.md +++ b/docs/API/chats.md @@ -116,6 +116,10 @@ The modified chat message This will return a list of chats that you have been involved in, sorted by their last update (so new chats will be at the top). +Parameters: + +- with_muted: Include chats from muted users (boolean). + Returned data: ```json diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex index 0dcfdb354..560b81f17 100644 --- a/lib/pleroma/web/api_spec/operations/chat_operation.ex +++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do alias OpenApiSpex.Operation alias OpenApiSpex.Schema alias Pleroma.Web.ApiSpec.Schemas.ApiError + alias Pleroma.Web.ApiSpec.Schemas.BooleanLike alias Pleroma.Web.ApiSpec.Schemas.Chat alias Pleroma.Web.ApiSpec.Schemas.ChatMessage @@ -132,7 +133,10 @@ def index_operation do tags: ["chat"], summary: "Get a list of chats that you participated in", operationId: "ChatController.index", - parameters: pagination_params(), + parameters: [ + Operation.parameter(:with_muted, :query, BooleanLike, "Include chats from muted users") + | pagination_params() + ], responses: %{ 200 => Operation.response("The chats of the user", "application/json", chats_response()) }, diff --git a/lib/pleroma/web/api_spec/operations/timeline_operation.ex b/lib/pleroma/web/api_spec/operations/timeline_operation.ex index 8e19bace7..1b5ad796f 100644 --- a/lib/pleroma/web/api_spec/operations/timeline_operation.ex +++ b/lib/pleroma/web/api_spec/operations/timeline_operation.ex @@ -159,7 +159,7 @@ defp local_param do end defp with_muted_param do - Operation.parameter(:with_muted, :query, BooleanLike, "Includeactivities by muted users") + Operation.parameter(:with_muted, :query, BooleanLike, "Include activities by muted users") end defp exclude_visibilities_param do diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex index 8fc70c15a..77564b342 100644 --- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex @@ -138,11 +138,10 @@ def messages(%{assigns: %{user: user}} = conn, %{id: id} = params) do end end - def index(%{assigns: %{user: %{id: user_id} = user}} = conn, _params) do + def index(%{assigns: %{user: %{id: user_id} = user}} = conn, params) do exclude_users = - user - |> User.blocked_users_ap_ids() - |> Enum.concat(User.muted_users_ap_ids(user)) + User.blocked_users_ap_ids(user) ++ + if params[:with_muted], do: [], else: User.muted_users_ap_ids(user) chats = user_id diff --git a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs index b0498df2b..c1e6a8cc5 100644 --- a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs @@ -363,6 +363,13 @@ test "it does not return chats with users you muted", %{conn: conn, user: user} |> json_response_and_validate_schema(200) assert length(result) == 0 + + result = + conn + |> get("/api/v1/pleroma/chats?with_muted=true") + |> json_response_and_validate_schema(200) + + assert length(result) == 1 end test "it returns all chats", %{conn: conn, user: user} do From cb3cd3a761d96a08b1b55f5b277795822aa7e1d7 Mon Sep 17 00:00:00 2001 From: lain <lain@soykaf.club> Date: Wed, 4 Nov 2020 15:24:10 +0100 Subject: [PATCH 40/74] TopicsTest: Small addition. --- test/pleroma/activity/ir/topics_test.exs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/pleroma/activity/ir/topics_test.exs b/test/pleroma/activity/ir/topics_test.exs index 2e5655334..5e5c2f8da 100644 --- a/test/pleroma/activity/ir/topics_test.exs +++ b/test/pleroma/activity/ir/topics_test.exs @@ -104,6 +104,13 @@ test "non-local action produces public:remote topic", %{activity: activity} do assert Enum.member?(topics, "public:remote:lain.com") end + + test "local action doesn't produce public:remote topic", %{activity: activity} do + activity = %{activity | local: true, actor: "https://lain.com/users/lain"} + topics = Topics.get_activity_topics(activity) + + refute Enum.member?(topics, "public:remote:lain.com") + end end describe "public visibility create events with attachments" do From eb1e1e74945a55c37e4b0dc7cea1a7c926ff981a Mon Sep 17 00:00:00 2001 From: lain <lain@soykaf.club> Date: Wed, 4 Nov 2020 15:39:32 +0100 Subject: [PATCH 41/74] Changelog: Add info about federation status endpoint --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ee17d239..8951b4523 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Admin API: (`GET /api/pleroma/admin/users`) added filters user by `unconfirmed` status - Admin API: (`GET /api/pleroma/admin/users`) added filters user by `actor_type` - Pleroma API: Add `idempotency_key` to the chat message entity that can be used for optimistic message sending. +- Pleroma API: (`GET /api/v1/pleroma/federation_status`) Add a way to get a list of unreachable instances. </details> From 6d850c46dc439e018323fb3580e708131247457b Mon Sep 17 00:00:00 2001 From: lain <lain@soykaf.club> Date: Wed, 4 Nov 2020 17:12:47 +0100 Subject: [PATCH 42/74] AdminEmail: Use AP id as user url. --- lib/pleroma/emails/admin_email.ex | 10 +++------- test/pleroma/emails/admin_email_test.exs | 6 +++--- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/emails/admin_email.ex b/lib/pleroma/emails/admin_email.ex index 8979db2f8..02274554f 100644 --- a/lib/pleroma/emails/admin_email.ex +++ b/lib/pleroma/emails/admin_email.ex @@ -18,10 +18,6 @@ defp instance_notify_email do Keyword.get(instance_config(), :notify_email, instance_config()[:email]) end - defp user_url(user) do - Helpers.user_feed_url(Pleroma.Web.Endpoint, :feed_redirect, user.id) - end - def test_email(mail_to \\ nil) do html_body = """ <h3>Instance Test Email</h3> @@ -69,8 +65,8 @@ def report(to, reporter, account, statuses, comment) do end html_body = """ - <p>Reported by: <a href="#{user_url(reporter)}">#{reporter.nickname}</a></p> - <p>Reported Account: <a href="#{user_url(account)}">#{account.nickname}</a></p> + <p>Reported by: <a href="#{reporter.ap_id}">#{reporter.nickname}</a></p> + <p>Reported Account: <a href="#{account.ap_id}">#{account.nickname}</a></p> #{comment_html} #{statuses_html} <p> @@ -86,7 +82,7 @@ def report(to, reporter, account, statuses, comment) do def new_unapproved_registration(to, account) do html_body = """ - <p>New account for review: <a href="#{user_url(account)}">@#{account.nickname}</a></p> + <p>New account for review: <a href="#{account.ap_id}">@#{account.nickname}</a></p> <blockquote>#{HTML.strip_tags(account.registration_reason)}</blockquote> <a href="#{Pleroma.Web.base_url()}/pleroma/admin/#/users/#{account.id}/">Visit AdminFE</a> """ diff --git a/test/pleroma/emails/admin_email_test.exs b/test/pleroma/emails/admin_email_test.exs index 155057f3e..0da0699cc 100644 --- a/test/pleroma/emails/admin_email_test.exs +++ b/test/pleroma/emails/admin_email_test.exs @@ -19,8 +19,8 @@ test "build report email" do AdminEmail.report(to_user, reporter, account, [%{name: "Test", id: "12"}], "Test comment") status_url = Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, "12") - reporter_url = Helpers.user_feed_url(Pleroma.Web.Endpoint, :feed_redirect, reporter.id) - account_url = Helpers.user_feed_url(Pleroma.Web.Endpoint, :feed_redirect, account.id) + reporter_url = reporter.ap_id + account_url = account.ap_id assert res.to == [{to_user.name, to_user.email}] assert res.from == {config[:name], config[:notify_email]} @@ -54,7 +54,7 @@ test "new unapproved registration email" do res = AdminEmail.new_unapproved_registration(to_user, account) - account_url = Helpers.user_feed_url(Pleroma.Web.Endpoint, :feed_redirect, account.id) + account_url = account.ap_id assert res.to == [{to_user.name, to_user.email}] assert res.from == {config[:name], config[:notify_email]} From 5ddf0be208b1a2703f24c907ac5a07cb8ea12f8c Mon Sep 17 00:00:00 2001 From: lain <lain@soykaf.club> Date: Wed, 4 Nov 2020 17:13:34 +0100 Subject: [PATCH 43/74] Changelog: Add info about admin email user link changes. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a82f7d6c..756939dde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option). - Introduced optional dependencies on `ffmpeg`, `ImageMagick`, `exiftool` software packages. Please refer to `docs/installation/optional/media_graphics_packages.md`. - Polls now always return a `voters_count`, even if they are single-choice +- Admin Emails: The ap id is used as the user link in emails now. <details> <summary>API Changes</summary> From 9b2ed1427725cd8c5e8d81e39cc5bbc73a2140bc Mon Sep 17 00:00:00 2001 From: lain <lain@soykaf.club> Date: Thu, 5 Nov 2020 13:23:58 +0100 Subject: [PATCH 44/74] Docs: Add info about expiring mutes. --- docs/API/differences_in_mastoapi_responses.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/API/differences_in_mastoapi_responses.md b/docs/API/differences_in_mastoapi_responses.md index bb1000b0b..3075b6b86 100644 --- a/docs/API/differences_in_mastoapi_responses.md +++ b/docs/API/differences_in_mastoapi_responses.md @@ -255,6 +255,10 @@ There is an additional `user:pleroma_chat` stream. Incoming chat messages will m For viewing remote server timelines, there are `public:remote` and `public:remote:media` streams. Each of these accept a parameter like `?instance=lain.com`. +## User muting and thread muting + +Both user muting and thread muting can be done for only a certain time by adding an `expires_in` parameter to the API calls and giving the expiration time in seconds. + ## Not implemented Pleroma is generally compatible with the Mastodon 2.7.2 API, but some newer features and non-essential features are omitted. These features usually return an HTTP 200 status code, but with an empty response. While they may be added in the future, they are considered low priority. From 60fe4a8393fe874284fbf2f46d598119a5207296 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@FreeBSD.org> Date: Fri, 6 Nov 2020 13:00:31 -0600 Subject: [PATCH 45/74] First draft of tips for optimizing BEAM --- docs/configuration/optimizing_beam.md | 60 +++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 docs/configuration/optimizing_beam.md diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md new file mode 100644 index 000000000..7a3cfda4b --- /dev/null +++ b/docs/configuration/optimizing_beam.md @@ -0,0 +1,60 @@ +# Optimizing the BEAM + +Pleroma is built upon the Erlang/OTP VM known as BEAM. The BEAM VM is highly optimized for latency, but this has drawbacks in environments without dedicated hardware. One of the tricks used by the BEAM VM is [busy waiting](https://en.wikipedia.org/wiki/Busy_waiting). This allows the application to pretend to be busy working so the OS kernel does not pause the application process and switch to another process waiting for the CPU to execute its workload. It does this by busy waiting or spinning for a period of time which inflates the apparent CPU usage of the application as seen in utilities like **top(1)** so it is immediately ready to execute another task. Switching between procesess is a rather expensive operation and also clears CPU caches which further affects latency and performance. + +This strategy is very successful in making a performant and responsive application, but is not desirable on Virtual Machines or hardware with few CPU cores. Pleroma instances are often deployed on the same server as the required PostgreSQL database which can lead to situations where the Pleroma application is holding the CPU in a busy-wait loop and as a result the database cannot process requests in a timely manner. The fewer CPUs available, the more this problem is exacerbated. The latency is further amplified by the OS being installed on a Virtual Machine as the Hypervisor uses CPU time-slicing to pause the entire OS and switch between other tasks. + +More adventerous admins can be creative with CPU affinity (e.g., *taskset* for Linux and *cpuset* on FreeBSD) to pin processes to specific CPUs and eliminate much of this contention. The most important advice is to run as few processes as possible on your server to achieve the best performance. Even idle background processes can occasionally create [software interrupts](https://en.wikipedia.org/wiki/Interrupt) and take attention away from the executing process and latency spikes and invalidation of the CPU caches as they must be cleared when switching between processes for security. + +## VPS Provider Recommendations + +### Good + +* ???? + +### Bad + +* AWS (known to use burst scheduling) + + +## Example configurations + +Tuning the BEAM requires you provide a config file normally called [vm.args](http://erlang.org/doc/man/erl.html#emulator-flags). If you are using systemd to manage the service you can modify the unit file as such: + +`ExecStart=/usr/bin/elixir --erl '-args_file /opt/pleroma/config/vm.args' -S /usr/bin/mix phx.server` + +Check your OS documentation to adopt a similar strategy on other platforms. + +### Virtual Machine and/or few CPU cores + +Disable the busy-waiting + +``` ++sbwt none ++sbwtdcpu none ++sbwtdio none +``` + +### Dedicated Hardware + +Enable more busy waiting, increase the internal maximum limit of BEAM processes and ports + +``` ++P 16777216 ++Q 16777216 ++K true ++A 128 ++sbt db ++sbwt very_long ++swt very_low ++sub true ++Mulmbcs 32767 ++Mumbcgs 1 ++Musmbcs 2047 +``` + +## Additional Reading + +* [WhatsApp: Scaling to Millions of Simultaneous Connections](https://www.erlang-factory.com/upload/presentations/558/efsf2012-whatsapp-scaling.pdf) +* [Preemptive Scheduling and Spinlocks](https://www.uio.no/studier/emner/matnat/ifi/nedlagte-emner/INF3150/h03/annet/slides/preemptive.pdf) +* [The Curious Case of BEAM CPU Usage](https://stressgrid.com/blog/beam_cpu_usage/) From 9e90e49ad2d01744bd5385473f2ebd4b2a442e8a Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@FreeBSD.org> Date: Fri, 6 Nov 2020 13:02:07 -0600 Subject: [PATCH 46/74] Grammar --- docs/configuration/optimizing_beam.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md index 7a3cfda4b..40fa7bfbf 100644 --- a/docs/configuration/optimizing_beam.md +++ b/docs/configuration/optimizing_beam.md @@ -4,7 +4,7 @@ Pleroma is built upon the Erlang/OTP VM known as BEAM. The BEAM VM is highly opt This strategy is very successful in making a performant and responsive application, but is not desirable on Virtual Machines or hardware with few CPU cores. Pleroma instances are often deployed on the same server as the required PostgreSQL database which can lead to situations where the Pleroma application is holding the CPU in a busy-wait loop and as a result the database cannot process requests in a timely manner. The fewer CPUs available, the more this problem is exacerbated. The latency is further amplified by the OS being installed on a Virtual Machine as the Hypervisor uses CPU time-slicing to pause the entire OS and switch between other tasks. -More adventerous admins can be creative with CPU affinity (e.g., *taskset* for Linux and *cpuset* on FreeBSD) to pin processes to specific CPUs and eliminate much of this contention. The most important advice is to run as few processes as possible on your server to achieve the best performance. Even idle background processes can occasionally create [software interrupts](https://en.wikipedia.org/wiki/Interrupt) and take attention away from the executing process and latency spikes and invalidation of the CPU caches as they must be cleared when switching between processes for security. +More adventerous admins can be creative with CPU affinity (e.g., *taskset* for Linux and *cpuset* on FreeBSD) to pin processes to specific CPUs and eliminate much of this contention. The most important advice is to run as few processes as possible on your server to achieve the best performance. Even idle background processes can occasionally create [software interrupts](https://en.wikipedia.org/wiki/Interrupt) and take attention away from the executing process creating latency spikes and invalidation of the CPU caches as they must be cleared when switching between processes for security. ## VPS Provider Recommendations From da1862e1d3fea5488b80bfc46ea878468940238c Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@FreeBSD.org> Date: Fri, 6 Nov 2020 13:04:13 -0600 Subject: [PATCH 47/74] Less confusing I hope --- docs/configuration/optimizing_beam.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md index 40fa7bfbf..00620b35d 100644 --- a/docs/configuration/optimizing_beam.md +++ b/docs/configuration/optimizing_beam.md @@ -1,6 +1,6 @@ # Optimizing the BEAM -Pleroma is built upon the Erlang/OTP VM known as BEAM. The BEAM VM is highly optimized for latency, but this has drawbacks in environments without dedicated hardware. One of the tricks used by the BEAM VM is [busy waiting](https://en.wikipedia.org/wiki/Busy_waiting). This allows the application to pretend to be busy working so the OS kernel does not pause the application process and switch to another process waiting for the CPU to execute its workload. It does this by busy waiting or spinning for a period of time which inflates the apparent CPU usage of the application as seen in utilities like **top(1)** so it is immediately ready to execute another task. Switching between procesess is a rather expensive operation and also clears CPU caches which further affects latency and performance. +Pleroma is built upon the Erlang/OTP VM known as BEAM. The BEAM VM is highly optimized for latency, but this has drawbacks in environments without dedicated hardware. One of the tricks used by the BEAM VM is [busy waiting](https://en.wikipedia.org/wiki/Busy_waiting). This allows the application to pretend to be busy working so the OS kernel does not pause the application process and switch to another process waiting for the CPU to execute its workload. It does this by spinning for a period of time which inflates the apparent CPU usage of the application as seen in utilities like **top(1)** so it is immediately ready to execute another task. Switching between procesess is a rather expensive operation and also clears CPU caches which further affects latency and performance. This strategy is very successful in making a performant and responsive application, but is not desirable on Virtual Machines or hardware with few CPU cores. Pleroma instances are often deployed on the same server as the required PostgreSQL database which can lead to situations where the Pleroma application is holding the CPU in a busy-wait loop and as a result the database cannot process requests in a timely manner. The fewer CPUs available, the more this problem is exacerbated. The latency is further amplified by the OS being installed on a Virtual Machine as the Hypervisor uses CPU time-slicing to pause the entire OS and switch between other tasks. From 620f1d723781ef39079add6326aec24112275253 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@FreeBSD.org> Date: Fri, 6 Nov 2020 13:12:13 -0600 Subject: [PATCH 48/74] More grammar fixes --- docs/configuration/optimizing_beam.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md index 00620b35d..fd89c4e99 100644 --- a/docs/configuration/optimizing_beam.md +++ b/docs/configuration/optimizing_beam.md @@ -1,6 +1,6 @@ # Optimizing the BEAM -Pleroma is built upon the Erlang/OTP VM known as BEAM. The BEAM VM is highly optimized for latency, but this has drawbacks in environments without dedicated hardware. One of the tricks used by the BEAM VM is [busy waiting](https://en.wikipedia.org/wiki/Busy_waiting). This allows the application to pretend to be busy working so the OS kernel does not pause the application process and switch to another process waiting for the CPU to execute its workload. It does this by spinning for a period of time which inflates the apparent CPU usage of the application as seen in utilities like **top(1)** so it is immediately ready to execute another task. Switching between procesess is a rather expensive operation and also clears CPU caches which further affects latency and performance. +Pleroma is built upon the Erlang/OTP VM known as BEAM. The BEAM VM is highly optimized for latency, but this has drawbacks in environments without dedicated hardware. One of the tricks used by the BEAM VM is [busy waiting](https://en.wikipedia.org/wiki/Busy_waiting). This allows the application to pretend to be busy working so the OS kernel does not pause the application process and switch to another process waiting for the CPU to execute its workload. It does this by spinning for a period of time which inflates the apparent CPU usage of the application so it is immediately ready to execute another task. This can be observed with utilities like **top(1)** which will show consistently high CPU usage for the process. Switching between procesess is a rather expensive operation and also clears CPU caches further affecting latency and performance. The goal of busy waiting is to avoid this penalty. This strategy is very successful in making a performant and responsive application, but is not desirable on Virtual Machines or hardware with few CPU cores. Pleroma instances are often deployed on the same server as the required PostgreSQL database which can lead to situations where the Pleroma application is holding the CPU in a busy-wait loop and as a result the database cannot process requests in a timely manner. The fewer CPUs available, the more this problem is exacerbated. The latency is further amplified by the OS being installed on a Virtual Machine as the Hypervisor uses CPU time-slicing to pause the entire OS and switch between other tasks. From 4999549191b1ac7c50bb3a6398b0bad0f0957b73 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@FreeBSD.org> Date: Fri, 6 Nov 2020 13:15:21 -0600 Subject: [PATCH 49/74] Make it clearer the settings go into the vm.args file --- docs/configuration/optimizing_beam.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md index fd89c4e99..64d35ad2c 100644 --- a/docs/configuration/optimizing_beam.md +++ b/docs/configuration/optimizing_beam.md @@ -29,6 +29,7 @@ Check your OS documentation to adopt a similar strategy on other platforms. Disable the busy-waiting +**vm.args:** ``` +sbwt none +sbwtdcpu none @@ -39,6 +40,7 @@ Disable the busy-waiting Enable more busy waiting, increase the internal maximum limit of BEAM processes and ports +**vm.args:** ``` +P 16777216 +Q 16777216 From a9c1f83fd8b2793e6474b14d246e1ef362892467 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@FreeBSD.org> Date: Fri, 6 Nov 2020 13:16:22 -0600 Subject: [PATCH 50/74] Markdown, you're drunk --- docs/configuration/optimizing_beam.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md index 64d35ad2c..de76086f7 100644 --- a/docs/configuration/optimizing_beam.md +++ b/docs/configuration/optimizing_beam.md @@ -30,6 +30,7 @@ Check your OS documentation to adopt a similar strategy on other platforms. Disable the busy-waiting **vm.args:** + ``` +sbwt none +sbwtdcpu none @@ -41,6 +42,7 @@ Disable the busy-waiting Enable more busy waiting, increase the internal maximum limit of BEAM processes and ports **vm.args:** + ``` +P 16777216 +Q 16777216 From cc45c69fff515cb82c4902b67c8edce6b109c819 Mon Sep 17 00:00:00 2001 From: rinpatch <rinpatch@sdf.org> Date: Sat, 7 Nov 2020 22:09:28 +0300 Subject: [PATCH 51/74] Remove release_env While taking a final look at instance.gen before releasing I noticed that the release_env task outputs messages in broken english. Upon further inspection it seems to have even more severe issues which, in my opinion, warrant it's at least temporary removal: - We do not explain what it actually does, anywhere. Neither the task docs nor instance.gen, nor installation instructions. - It does not respect FHS on OTP releases (uses /opt/pleroma/config even though we store the config in /etc/pleroma/config.exs). - It doesn't work on OTP releases, which is the main reason it exists. Neither systemd nor openrc service files for OTP include it. - It is not mentioned in install guides other than the ones for Debian and OTP releases. --- .../CLI_tasks/release_environments.md | 9 --- docs/installation/debian_based_en.md | 1 - docs/installation/otp_en.md | 3 - installation/init.d/pleroma | 1 - installation/pleroma.service | 2 - lib/mix/tasks/pleroma/instance.ex | 22 +----- lib/mix/tasks/pleroma/release_env.ex | 76 ------------------- test/mix/tasks/pleroma/instance_test.exs | 11 +-- test/mix/tasks/pleroma/release_env_test.exs | 30 -------- 9 files changed, 2 insertions(+), 153 deletions(-) delete mode 100644 docs/administration/CLI_tasks/release_environments.md delete mode 100644 lib/mix/tasks/pleroma/release_env.ex delete mode 100644 test/mix/tasks/pleroma/release_env_test.exs diff --git a/docs/administration/CLI_tasks/release_environments.md b/docs/administration/CLI_tasks/release_environments.md deleted file mode 100644 index 36ab43864..000000000 --- a/docs/administration/CLI_tasks/release_environments.md +++ /dev/null @@ -1,9 +0,0 @@ -# Generate release environment file - -```sh tab="OTP" - ./bin/pleroma_ctl release_env gen -``` - -```sh tab="From Source" -mix pleroma.release_env gen -``` diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md index b9fc4e112..75ceb6595 100644 --- a/docs/installation/debian_based_en.md +++ b/docs/installation/debian_based_en.md @@ -182,7 +182,6 @@ sudo cp /opt/pleroma/installation/pleroma.service /etc/systemd/system/pleroma.se ``` * Edit the service file and make sure that all paths fit your installation -* Check that `EnvironmentFile` contains the correct path to the env file. Or generate the env file: `sudo -Hu pleroma mix pleroma.release_env gen` * Enable and start `pleroma.service`: ```shell diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index 98360bcf7..63eda63ca 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -149,9 +149,6 @@ chown -R pleroma /etc/pleroma # Run the config generator su pleroma -s $SHELL -lc "./bin/pleroma_ctl instance gen --output /etc/pleroma/config.exs --output-psql /tmp/setup_db.psql" -# Run the environment file generator. -su pleroma -s $SHELL -lc "./bin/pleroma_ctl release_env gen" - # Create the postgres database su postgres -s $SHELL -lc "psql -f /tmp/setup_db.psql" diff --git a/installation/init.d/pleroma b/installation/init.d/pleroma index e908cda1b..384536f7e 100755 --- a/installation/init.d/pleroma +++ b/installation/init.d/pleroma @@ -8,7 +8,6 @@ pidfile="/var/run/pleroma.pid" directory=/opt/pleroma healthcheck_delay=60 healthcheck_timer=30 -export $(cat /opt/pleroma/config/pleroma.env) : ${pleroma_port:-4000} diff --git a/installation/pleroma.service b/installation/pleroma.service index 63e83ed6e..8338228d8 100644 --- a/installation/pleroma.service +++ b/installation/pleroma.service @@ -17,8 +17,6 @@ Environment="MIX_ENV=prod" Environment="HOME=/var/lib/pleroma" ; Path to the folder containing the Pleroma installation. WorkingDirectory=/opt/pleroma -; Path to the environment file. the file contains RELEASE_COOKIE and etc -EnvironmentFile=/opt/pleroma/config/pleroma.env ; Path to the Mix binary. ExecStart=/usr/bin/mix phx.server diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index 1915aacd9..fc21ae062 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -36,9 +36,7 @@ def run(["gen" | rest]) do listen_port: :string, strip_uploads: :string, anonymize_uploads: :string, - dedupe_uploads: :string, - skip_release_env: :boolean, - release_env_file: :string + dedupe_uploads: :string ], aliases: [ o: :output, @@ -243,24 +241,6 @@ def run(["gen" | rest]) do write_robots_txt(static_dir, indexable, template_dir) - if Keyword.get(options, :skip_release_env, false) do - shell_info(""" - Release environment file is skip. Please generate the release env file before start. - `MIX_ENV=#{Mix.env()} mix pleroma.release_env gen` - """) - else - shell_info("Generation the environment file:") - - release_env_args = - with path when not is_nil(path) <- Keyword.get(options, :release_env_file) do - ["gen", "--path", path] - else - _ -> ["gen"] - end - - Mix.Tasks.Pleroma.ReleaseEnv.run(release_env_args) - end - shell_info( "\n All files successfully written! Refer to the installation instructions for your platform for next steps." ) diff --git a/lib/mix/tasks/pleroma/release_env.ex b/lib/mix/tasks/pleroma/release_env.ex deleted file mode 100644 index 9da74ffcf..000000000 --- a/lib/mix/tasks/pleroma/release_env.ex +++ /dev/null @@ -1,76 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Mix.Tasks.Pleroma.ReleaseEnv do - use Mix.Task - import Mix.Pleroma - - @shortdoc "Generate Pleroma environment file." - @moduledoc File.read!("docs/administration/CLI_tasks/release_environments.md") - - def run(["gen" | rest]) do - {options, [], []} = - OptionParser.parse( - rest, - strict: [ - force: :boolean, - path: :string - ], - aliases: [ - p: :path, - f: :force - ] - ) - - file_path = - get_option( - options, - :path, - "Environment file path", - "./config/pleroma.env" - ) - - env_path = Path.expand(file_path) - - proceed? = - if File.exists?(env_path) do - get_option( - options, - :force, - "Environment file already exists. Do you want to overwrite the #{env_path} file? (y/n)", - "n" - ) === "y" - else - true - end - - if proceed? do - case do_generate(env_path) do - {:error, reason} -> - shell_error( - File.Error.message(%{action: "write to file", reason: reason, path: env_path}) - ) - - _ -> - shell_info("\nThe file generated: #{env_path}.\n") - - shell_info(""" - WARNING: before start pleroma app please make sure to make the file read-only and non-modifiable. - Example: - chmod 0444 #{file_path} - chattr +i #{file_path} - """) - end - else - shell_info("\nThe file is exist. #{env_path}.\n") - end - end - - def do_generate(path) do - content = "RELEASE_COOKIE=#{Base.encode32(:crypto.strong_rand_bytes(32))}" - - File.mkdir_p!(Path.dirname(path)) - File.write(path, content) - end -end diff --git a/test/mix/tasks/pleroma/instance_test.exs b/test/mix/tasks/pleroma/instance_test.exs index fe69a2def..8a02710ee 100644 --- a/test/mix/tasks/pleroma/instance_test.exs +++ b/test/mix/tasks/pleroma/instance_test.exs @@ -5,8 +5,6 @@ defmodule Mix.Tasks.Pleroma.InstanceTest do use ExUnit.Case - @release_env_file "./test/pleroma.test.env" - setup do File.mkdir_p!(tmp_path()) @@ -18,8 +16,6 @@ defmodule Mix.Tasks.Pleroma.InstanceTest do File.rm_rf(Path.join(static_dir, "robots.txt")) end - if File.exists?(@release_env_file), do: File.rm_rf(@release_env_file) - Pleroma.Config.put([:instance, :static_dir], static_dir) end) @@ -73,9 +69,7 @@ test "running gen" do "--dedupe-uploads", "n", "--anonymize-uploads", - "n", - "--release-env-file", - @release_env_file + "n" ]) end @@ -97,9 +91,6 @@ test "running gen" do assert generated_config =~ "filters: [Pleroma.Upload.Filter.ExifTool]" assert File.read!(tmp_path() <> "setup.psql") == generated_setup_psql() assert File.exists?(Path.expand("./test/instance/static/robots.txt")) - assert File.exists?(@release_env_file) - - assert File.read!(@release_env_file) =~ ~r/^RELEASE_COOKIE=.*/ end defp generated_setup_psql do diff --git a/test/mix/tasks/pleroma/release_env_test.exs b/test/mix/tasks/pleroma/release_env_test.exs deleted file mode 100644 index 519f1eba9..000000000 --- a/test/mix/tasks/pleroma/release_env_test.exs +++ /dev/null @@ -1,30 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Mix.Tasks.Pleroma.ReleaseEnvTest do - use ExUnit.Case - import ExUnit.CaptureIO, only: [capture_io: 1] - - @path "config/pleroma.test.env" - - def do_clean do - if File.exists?(@path) do - File.rm_rf(@path) - end - end - - setup do - do_clean() - on_exit(fn -> do_clean() end) - :ok - end - - test "generate pleroma.env" do - assert capture_io(fn -> - Mix.Tasks.Pleroma.ReleaseEnv.run(["gen", "--path", @path, "--force"]) - end) =~ "The file generated" - - assert File.read!(@path) =~ "RELEASE_COOKIE=" - end -end From abf2ec2bbe5226d6558ca1918779ee68df1cab84 Mon Sep 17 00:00:00 2001 From: lain <lain@soykaf.club> Date: Sun, 8 Nov 2020 09:45:35 +0000 Subject: [PATCH 52/74] Update optimizing_beam.md --- docs/configuration/optimizing_beam.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md index de76086f7..545392f8f 100644 --- a/docs/configuration/optimizing_beam.md +++ b/docs/configuration/optimizing_beam.md @@ -4,7 +4,7 @@ Pleroma is built upon the Erlang/OTP VM known as BEAM. The BEAM VM is highly opt This strategy is very successful in making a performant and responsive application, but is not desirable on Virtual Machines or hardware with few CPU cores. Pleroma instances are often deployed on the same server as the required PostgreSQL database which can lead to situations where the Pleroma application is holding the CPU in a busy-wait loop and as a result the database cannot process requests in a timely manner. The fewer CPUs available, the more this problem is exacerbated. The latency is further amplified by the OS being installed on a Virtual Machine as the Hypervisor uses CPU time-slicing to pause the entire OS and switch between other tasks. -More adventerous admins can be creative with CPU affinity (e.g., *taskset* for Linux and *cpuset* on FreeBSD) to pin processes to specific CPUs and eliminate much of this contention. The most important advice is to run as few processes as possible on your server to achieve the best performance. Even idle background processes can occasionally create [software interrupts](https://en.wikipedia.org/wiki/Interrupt) and take attention away from the executing process creating latency spikes and invalidation of the CPU caches as they must be cleared when switching between processes for security. +More adventurous admins can be creative with CPU affinity (e.g., *taskset* for Linux and *cpuset* on FreeBSD) to pin processes to specific CPUs and eliminate much of this contention. The most important advice is to run as few processes as possible on your server to achieve the best performance. Even idle background processes can occasionally create [software interrupts](https://en.wikipedia.org/wiki/Interrupt) and take attention away from the executing process creating latency spikes and invalidation of the CPU caches as they must be cleared when switching between processes for security. ## VPS Provider Recommendations From e4a21084f0017c873a307ec85d0ef5ea341ce026 Mon Sep 17 00:00:00 2001 From: Sean King <seanking2919@protonmail.com> Date: Sun, 8 Nov 2020 16:16:20 -0700 Subject: [PATCH 53/74] Fix title on load of Pleroma HTML --- lib/pleroma/web/fallback/redirect_controller.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pleroma/web/fallback/redirect_controller.ex b/lib/pleroma/web/fallback/redirect_controller.ex index 6f759d559..712991c18 100644 --- a/lib/pleroma/web/fallback/redirect_controller.ex +++ b/lib/pleroma/web/fallback/redirect_controller.ex @@ -37,9 +37,11 @@ def redirector_with_meta(conn, params) do tags = build_tags(conn, params) preloads = preload_data(conn, params) + title = "<title>#{Pleroma.Config.get([:instance, :name])}</title>" response = index_content + |> String.replace(~r/<title>.+?<\/title>/, title) |> String.replace("<!--server-generated-meta-->", tags <> preloads) conn @@ -54,9 +56,11 @@ def redirector_with_preload(conn, %{"path" => ["pleroma", "admin"]}) do def redirector_with_preload(conn, params) do {:ok, index_content} = File.read(index_file_path()) preloads = preload_data(conn, params) + title = "<title>#{Pleroma.Config.get([:instance, :name])}</title>" response = index_content + |> String.replace(~r/<title>.+?<\/title>/, title) |> String.replace("<!--server-generated-meta-->", preloads) conn From 0c68b9ac137e98867cf8aacfef1f264412cc7b3e Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Tue, 10 Nov 2020 10:44:22 +0300 Subject: [PATCH 54/74] escaping summary and other fields in xml templates --- lib/pleroma/web/feed/feed_view.ex | 2 +- .../templates/feed/feed/_activity.atom.eex | 2 +- .../web/templates/feed/feed/_activity.rss.eex | 2 +- .../pleroma/web/feed/user_controller_test.exs | 77 +++++++------------ 4 files changed, 29 insertions(+), 54 deletions(-) diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex index 1ae03e7e2..56c024617 100644 --- a/lib/pleroma/web/feed/feed_view.ex +++ b/lib/pleroma/web/feed/feed_view.ex @@ -83,7 +83,7 @@ def activity_content(%{"content" => content}) do def activity_content(_), do: "" - def activity_context(activity), do: activity.data["context"] + def activity_context(activity), do: escape(activity.data["context"]) def attachment_href(attachment) do attachment["url"] diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex index 78350f2aa..3fd150c4e 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex @@ -12,7 +12,7 @@ <link href="<%= activity_context(@activity) %>" rel="ostatus:conversation"/> <%= if @data["summary"] do %> - <summary><%= @data["summary"] %></summary> + <summary><%= escape(@data["summary"]) %></summary> <% end %> <%= if @activity.local do %> diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex index a304a16af..42960de7d 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex @@ -12,7 +12,7 @@ <link rel="ostatus:conversation"><%= activity_context(@activity) %></link> <%= if @data["summary"] do %> - <description><%= @data["summary"] %></description> + <description><%= escape(@data["summary"]) %></description> <% end %> <%= if @activity.local do %> diff --git a/test/pleroma/web/feed/user_controller_test.exs b/test/pleroma/web/feed/user_controller_test.exs index eabfe3a63..16f002717 100644 --- a/test/pleroma/web/feed/user_controller_test.exs +++ b/test/pleroma/web/feed/user_controller_test.exs @@ -12,16 +12,17 @@ defmodule Pleroma.Web.Feed.UserControllerTest do alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.CommonAPI + alias Pleroma.Web.Feed.FeedView setup do: clear_config([:static_fe, :enabled], false) describe "feed" do setup do: clear_config([:feed]) - test "gets an atom feed", %{conn: conn} do + setup do Config.put( [:feed, :post_title], - %{max_length: 10, omission: "..."} + %{max_length: 15, omission: "..."} ) activity = insert(:note_activity) @@ -29,7 +30,8 @@ test "gets an atom feed", %{conn: conn} do note = insert(:note, data: %{ - "content" => "This is :moominmamma: note ", + "content" => "This & this is :moominmamma: note ", + "source" => "This & this is :moominmamma: note ", "attachment" => [ %{ "url" => [ @@ -37,7 +39,9 @@ test "gets an atom feed", %{conn: conn} do ] } ], - "inReplyTo" => activity.data["id"] + "inReplyTo" => activity.data["id"], + "context" => "2hu & as", + "summary" => "2hu & as" } ) @@ -48,7 +52,7 @@ test "gets an atom feed", %{conn: conn} do insert(:note, user: user, data: %{ - "content" => "42 This is :moominmamma: note ", + "content" => "42 & This is :moominmamma: note ", "inReplyTo" => activity.data["id"] } ) @@ -56,6 +60,10 @@ test "gets an atom feed", %{conn: conn} do note_activity2 = insert(:note_activity, note: note2) object = Object.normalize(note_activity) + [user: user, object: object, max_id: note_activity2.id] + end + + test "gets an atom feed", %{conn: conn, user: user, object: object, max_id: max_id} do resp = conn |> put_req_header("accept", "application/atom+xml") @@ -67,13 +75,15 @@ test "gets an atom feed", %{conn: conn} do |> SweetXml.parse() |> SweetXml.xpath(~x"//entry/title/text()"l) - assert activity_titles == ['42 This...', 'This is...'] - assert resp =~ object.data["content"] + assert activity_titles == ['42 & Thi...', 'This & t...'] + assert resp =~ FeedView.escape(object.data["content"]) + assert resp =~ FeedView.escape(object.data["summary"]) + assert resp =~ FeedView.escape(object.data["context"]) resp = conn |> put_req_header("accept", "application/atom+xml") - |> get("/users/#{user.nickname}/feed", %{"max_id" => note_activity2.id}) + |> get("/users/#{user.nickname}/feed", %{"max_id" => max_id}) |> response(200) activity_titles = @@ -81,47 +91,10 @@ test "gets an atom feed", %{conn: conn} do |> SweetXml.parse() |> SweetXml.xpath(~x"//entry/title/text()"l) - assert activity_titles == ['This is...'] + assert activity_titles == ['This & t...'] end - test "gets a rss feed", %{conn: conn} do - Pleroma.Config.put( - [:feed, :post_title], - %{max_length: 10, omission: "..."} - ) - - activity = insert(:note_activity) - - note = - insert(:note, - data: %{ - "content" => "This is :moominmamma: note ", - "attachment" => [ - %{ - "url" => [ - %{"mediaType" => "image/png", "href" => "https://pleroma.gov/image.png"} - ] - } - ], - "inReplyTo" => activity.data["id"] - } - ) - - note_activity = insert(:note_activity, note: note) - user = User.get_cached_by_ap_id(note_activity.data["actor"]) - - note2 = - insert(:note, - user: user, - data: %{ - "content" => "42 This is :moominmamma: note ", - "inReplyTo" => activity.data["id"] - } - ) - - note_activity2 = insert(:note_activity, note: note2) - object = Object.normalize(note_activity) - + test "gets a rss feed", %{conn: conn, user: user, object: object, max_id: max_id} do resp = conn |> put_req_header("accept", "application/rss+xml") @@ -133,13 +106,15 @@ test "gets a rss feed", %{conn: conn} do |> SweetXml.parse() |> SweetXml.xpath(~x"//item/title/text()"l) - assert activity_titles == ['42 This...', 'This is...'] - assert resp =~ object.data["content"] + assert activity_titles == ['42 & Thi...', 'This & t...'] + assert resp =~ FeedView.escape(object.data["content"]) + assert resp =~ FeedView.escape(object.data["summary"]) + assert resp =~ FeedView.escape(object.data["context"]) resp = conn |> put_req_header("accept", "application/rss+xml") - |> get("/users/#{user.nickname}/feed.rss", %{"max_id" => note_activity2.id}) + |> get("/users/#{user.nickname}/feed.rss", %{"max_id" => max_id}) |> response(200) activity_titles = @@ -147,7 +122,7 @@ test "gets a rss feed", %{conn: conn} do |> SweetXml.parse() |> SweetXml.xpath(~x"//item/title/text()"l) - assert activity_titles == ['This is...'] + assert activity_titles == ['This & t...'] end test "returns 404 for a missing feed", %{conn: conn} do From 485697d96c6a45127af22b9a5f357c3802dba73c Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Tue, 10 Nov 2020 19:18:53 +0300 Subject: [PATCH 55/74] config descriptions for custom MRF policies --- config/description.exs | 302 ------------------ lib/pleroma/docs/json.ex | 6 +- lib/pleroma/web/activity_pub/mrf.ex | 58 +++- .../mrf/activity_expiration_policy.ex | 18 ++ .../web/activity_pub/mrf/hellthread_policy.ex | 27 ++ .../web/activity_pub/mrf/keyword_policy.ex | 42 +++ .../web/activity_pub/mrf/mention_policy.ex | 18 ++ .../web/activity_pub/mrf/normalize_markup.ex | 19 ++ .../web/activity_pub/mrf/object_age_policy.ex | 28 ++ .../web/activity_pub/mrf/reject_non_public.ex | 23 ++ .../web/activity_pub/mrf/simple_policy.ex | 74 +++++ .../web/activity_pub/mrf/subchain_policy.ex | 24 ++ .../mrf/user_allow_list_policy.ex | 14 + .../web/activity_pub/mrf/vocabulary_policy.ex | 28 ++ test/fixtures/modules/good_mrf.ex | 19 ++ test/pleroma/web/activity_pub/mrf_test.exs | 17 + 16 files changed, 412 insertions(+), 305 deletions(-) create mode 100644 test/fixtures/modules/good_mrf.ex diff --git a/config/description.exs b/config/description.exs index 0b651696b..69072312c 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1,5 +1,4 @@ use Mix.Config -alias Pleroma.Docs.Generator websocket_config = [ path: "/websocket", @@ -1589,264 +1588,6 @@ } ] }, - %{ - group: :pleroma, - key: :mrf_simple, - tab: :mrf, - related_policy: "Pleroma.Web.ActivityPub.MRF.SimplePolicy", - label: "MRF Simple", - type: :group, - description: "Simple ingress policies", - children: [ - %{ - key: :media_removal, - type: {:list, :string}, - description: "List of instances to strip media attachments from", - suggestions: ["example.com", "*.example.com"] - }, - %{ - key: :media_nsfw, - label: "Media NSFW", - type: {:list, :string}, - description: "List of instances to tag all media as NSFW (sensitive) from", - suggestions: ["example.com", "*.example.com"] - }, - %{ - key: :federated_timeline_removal, - type: {:list, :string}, - description: - "List of instances to remove from the Federated (aka The Whole Known Network) Timeline", - suggestions: ["example.com", "*.example.com"] - }, - %{ - key: :reject, - type: {:list, :string}, - description: "List of instances to reject activities from (except deletes)", - suggestions: ["example.com", "*.example.com"] - }, - %{ - key: :accept, - type: {:list, :string}, - description: "List of instances to only accept activities from (except deletes)", - suggestions: ["example.com", "*.example.com"] - }, - %{ - key: :followers_only, - type: {:list, :string}, - description: "Force posts from the given instances to be visible by followers only", - suggestions: ["example.com", "*.example.com"] - }, - %{ - key: :report_removal, - type: {:list, :string}, - description: "List of instances to reject reports from", - suggestions: ["example.com", "*.example.com"] - }, - %{ - key: :avatar_removal, - type: {:list, :string}, - description: "List of instances to strip avatars from", - suggestions: ["example.com", "*.example.com"] - }, - %{ - key: :banner_removal, - type: {:list, :string}, - description: "List of instances to strip banners from", - suggestions: ["example.com", "*.example.com"] - }, - %{ - key: :reject_deletes, - type: {:list, :string}, - description: "List of instances to reject deletions from", - suggestions: ["example.com", "*.example.com"] - } - ] - }, - %{ - group: :pleroma, - key: :mrf_activity_expiration, - tab: :mrf, - related_policy: "Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy", - label: "MRF Activity Expiration Policy", - type: :group, - description: "Adds automatic expiration to all local activities", - children: [ - %{ - key: :days, - type: :integer, - description: "Default global expiration time for all local activities (in days)", - suggestions: [90, 365] - } - ] - }, - %{ - group: :pleroma, - key: :mrf_subchain, - tab: :mrf, - related_policy: "Pleroma.Web.ActivityPub.MRF.SubchainPolicy", - label: "MRF Subchain", - type: :group, - description: - "This policy processes messages through an alternate pipeline when a given message matches certain criteria." <> - " All criteria are configured as a map of regular expressions to lists of policy modules.", - children: [ - %{ - key: :match_actor, - type: {:map, {:list, :string}}, - description: "Matches a series of regular expressions against the actor field", - suggestions: [ - %{ - ~r/https:\/\/example.com/s => [Pleroma.Web.ActivityPub.MRF.DropPolicy] - } - ] - } - ] - }, - %{ - group: :pleroma, - key: :mrf_rejectnonpublic, - tab: :mrf, - related_policy: "Pleroma.Web.ActivityPub.MRF.RejectNonPublic", - description: "RejectNonPublic drops posts with non-public visibility settings.", - label: "MRF Reject Non Public", - type: :group, - children: [ - %{ - key: :allow_followersonly, - label: "Allow followers-only", - type: :boolean, - description: "Whether to allow followers-only posts" - }, - %{ - key: :allow_direct, - type: :boolean, - description: "Whether to allow direct messages" - } - ] - }, - %{ - group: :pleroma, - key: :mrf_hellthread, - tab: :mrf, - related_policy: "Pleroma.Web.ActivityPub.MRF.HellthreadPolicy", - label: "MRF Hellthread", - type: :group, - description: "Block messages with excessive user mentions", - children: [ - %{ - key: :delist_threshold, - type: :integer, - description: - "Number of mentioned users after which the message gets removed from timelines and" <> - "disables notifications. Set to 0 to disable.", - suggestions: [10] - }, - %{ - key: :reject_threshold, - type: :integer, - description: - "Number of mentioned users after which the messaged gets rejected. Set to 0 to disable.", - suggestions: [20] - } - ] - }, - %{ - group: :pleroma, - key: :mrf_keyword, - tab: :mrf, - related_policy: "Pleroma.Web.ActivityPub.MRF.KeywordPolicy", - label: "MRF Keyword", - type: :group, - description: - "Reject or Word-Replace messages matching a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html).", - children: [ - %{ - key: :reject, - type: {:list, :string}, - description: """ - A list of patterns which result in message being rejected. - - Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. - """, - suggestions: ["foo", ~r/foo/iu] - }, - %{ - key: :federated_timeline_removal, - type: {:list, :string}, - description: """ - A list of patterns which result in message being removed from federated timelines (a.k.a unlisted). - - Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. - """, - suggestions: ["foo", ~r/foo/iu] - }, - %{ - key: :replace, - type: {:list, :tuple}, - description: """ - **Pattern**: a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. - - **Replacement**: a string. Leaving the field empty is permitted. - """ - } - ] - }, - %{ - group: :pleroma, - key: :mrf_mention, - tab: :mrf, - related_policy: "Pleroma.Web.ActivityPub.MRF.MentionPolicy", - label: "MRF Mention", - type: :group, - description: "Block messages which mention a specific user", - children: [ - %{ - key: :actors, - type: {:list, :string}, - description: "A list of actors for which any post mentioning them will be dropped", - suggestions: ["actor1", "actor2"] - } - ] - }, - %{ - group: :pleroma, - key: :mrf_vocabulary, - tab: :mrf, - related_policy: "Pleroma.Web.ActivityPub.MRF.VocabularyPolicy", - label: "MRF Vocabulary", - type: :group, - description: "Filter messages which belong to certain activity vocabularies", - children: [ - %{ - key: :accept, - type: {:list, :string}, - description: - "A list of ActivityStreams terms to accept. If empty, all supported messages are accepted.", - suggestions: ["Create", "Follow", "Mention", "Announce", "Like"] - }, - %{ - key: :reject, - type: {:list, :string}, - description: - "A list of ActivityStreams terms to reject. If empty, no messages are rejected.", - suggestions: ["Create", "Follow", "Mention", "Announce", "Like"] - } - ] - }, - # %{ - # group: :pleroma, - # key: :mrf_user_allowlist, - # tab: :mrf, - # related_policy: "Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy", - # type: :map, - # description: - # "The keys in this section are the domain names that the policy should apply to." <> - # " Each key should be assigned a list of users that should be allowed through by their ActivityPub ID", - # suggestions: [ - # %{"example.org" => ["https://example.org/users/admin"]} - # ] - # ] - # }, %{ group: :pleroma, key: :media_proxy, @@ -3159,22 +2900,6 @@ } ] }, - %{ - group: :pleroma, - key: :mrf_normalize_markup, - tab: :mrf, - related_policy: "Pleroma.Web.ActivityPub.MRF.NormalizeMarkup", - label: "MRF Normalize Markup", - description: "MRF NormalizeMarkup settings. Scrub configured hypertext markup.", - type: :group, - children: [ - %{ - key: :scrub_policy, - type: :module, - suggestions: [Pleroma.HTML.Scrubber.Default] - } - ] - }, %{ group: :pleroma, key: Pleroma.User, @@ -3364,33 +3089,6 @@ } ] }, - %{ - group: :pleroma, - key: :mrf_object_age, - tab: :mrf, - related_policy: "Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy", - label: "MRF Object Age", - type: :group, - description: - "Rejects or delists posts based on their timestamp deviance from your server's clock.", - children: [ - %{ - key: :threshold, - type: :integer, - description: "Required age (in seconds) of a post before actions are taken.", - suggestions: [172_800] - }, - %{ - key: :actions, - type: {:list, :atom}, - description: - "A list of actions to apply to the post. `:delist` removes the post from public timelines; " <> - "`:strip_followers` removes followers from the ActivityPub recipient list ensuring they won't be delivered to home timelines; " <> - "`:reject` rejects the message entirely", - suggestions: [:delist, :strip_followers, :reject] - } - ] - }, %{ group: :pleroma, key: :modules, diff --git a/lib/pleroma/docs/json.ex b/lib/pleroma/docs/json.ex index 13618b509..a583e2a5b 100644 --- a/lib/pleroma/docs/json.ex +++ b/lib/pleroma/docs/json.ex @@ -11,7 +11,11 @@ defmodule Pleroma.Docs.JSON do @spec compile :: :ok def compile do - :persistent_term.put(@term, Pleroma.Docs.Generator.convert_to_strings(@raw_descriptions)) + descriptions = + Pleroma.Web.ActivityPub.MRF.config_descriptions() + |> Enum.reduce(@raw_descriptions, fn description, acc -> [description | acc] end) + + :persistent_term.put(@term, Pleroma.Docs.Generator.convert_to_strings(descriptions)) end @spec compiled_descriptions :: Map.t() diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 5e5361082..656e4c7ca 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -3,7 +3,26 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF do + require Logger + + @default_description %{ + label: "", + description: "", + children: [] + } + + @required_description_keys [:key, :related_policy] + @callback filter(Map.t()) :: {:ok | :reject, Map.t()} + @callback describe() :: {:ok | :error, Map.t()} + @callback config_description() :: %{ + optional(:children) => [map()], + key: atom(), + related_policy: String.t(), + label: String.t(), + description: String.t() + } + @optional_callbacks config_description: 0 def filter(policies, %{} = message) do policies @@ -51,8 +70,6 @@ def subdomain_match?(domains, host) do Enum.any?(domains, fn domain -> Regex.match?(domain, host) end) end - @callback describe() :: {:ok | :error, Map.t()} - def describe(policies) do {:ok, policy_configs} = policies @@ -82,4 +99,41 @@ def describe(policies) do end def describe, do: get_policies() |> describe() + + def config_descriptions do + Pleroma.Web.ActivityPub.MRF + |> Pleroma.Docs.Generator.list_behaviour_implementations() + |> config_descriptions() + end + + def config_descriptions(policies) do + Enum.reduce(policies, [], fn policy, acc -> + if function_exported?(policy, :config_description, 0) do + description = + @default_description + |> Map.merge(policy.config_description) + |> Map.put(:group, :pleroma) + |> Map.put(:tab, :mrf) + |> Map.put(:type, :group) + + if Enum.all?(@required_description_keys, &Map.has_key?(description, &1)) do + [description | acc] + else + Logger.warn( + "#{policy} config description doesn't have one or all required keys #{ + inspect(@required_description_keys) + }" + ) + + acc + end + else + Logger.info( + "#{policy} is excluded from config descriptions, because does not implement `config_description/0` method." + ) + + acc + end + end) + end end diff --git a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex index bee47b4ed..655a2ced0 100644 --- a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex @@ -40,4 +40,22 @@ defp maybe_add_expiration(activity) do _ -> Map.put(activity, "expires_at", expires_at) end end + + @impl true + def config_description do + %{ + key: :mrf_activity_expiration, + related_policy: "Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy", + label: "MRF Activity Expiration Policy", + description: "Adds automatic expiration to all local activities", + children: [ + %{ + key: :days, + type: :integer, + description: "Default global expiration time for all local activities (in days)", + suggestions: [90, 365] + } + ] + } + end end diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index 9ba07b4e3..3fd5c1e0a 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -97,4 +97,31 @@ def filter(message), do: {:ok, message} @impl true def describe, do: {:ok, %{mrf_hellthread: Pleroma.Config.get(:mrf_hellthread) |> Enum.into(%{})}} + + @impl true + def config_description do + %{ + key: :mrf_hellthread, + related_policy: "Pleroma.Web.ActivityPub.MRF.HellthreadPolicy", + label: "MRF Hellthread", + description: "Block messages with excessive user mentions", + children: [ + %{ + key: :delist_threshold, + type: :integer, + description: + "Number of mentioned users after which the message gets removed from timelines and" <> + "disables notifications. Set to 0 to disable.", + suggestions: [10] + }, + %{ + key: :reject_threshold, + type: :integer, + description: + "Number of mentioned users after which the messaged gets rejected. Set to 0 to disable.", + suggestions: [20] + } + ] + } + end end diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index db66cfa3e..ded0fe7f2 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -126,4 +126,46 @@ def describe do {:ok, %{mrf_keyword: mrf_keyword}} end + + @impl true + def config_description do + %{ + key: :mrf_keyword, + related_policy: "Pleroma.Web.ActivityPub.MRF.KeywordPolicy", + label: "MRF Keyword", + description: + "Reject or Word-Replace messages matching a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html).", + children: [ + %{ + key: :reject, + type: {:list, :string}, + description: """ + A list of patterns which result in message being rejected. + + Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. + """, + suggestions: ["foo", ~r/foo/iu] + }, + %{ + key: :federated_timeline_removal, + type: {:list, :string}, + description: """ + A list of patterns which result in message being removed from federated timelines (a.k.a unlisted). + + Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. + """, + suggestions: ["foo", ~r/foo/iu] + }, + %{ + key: :replace, + type: {:list, :tuple}, + description: """ + **Pattern**: a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. + + **Replacement**: a string. Leaving the field empty is permitted. + """ + } + ] + } + end end diff --git a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex index 7910ca131..9c096712a 100644 --- a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex @@ -25,4 +25,22 @@ def filter(message), do: {:ok, message} @impl true def describe, do: {:ok, %{}} + + @impl true + def config_description do + %{ + key: :mrf_mention, + related_policy: "Pleroma.Web.ActivityPub.MRF.MentionPolicy", + label: "MRF Mention", + description: "Block messages which mention a specific user", + children: [ + %{ + key: :actors, + type: {:list, :string}, + description: "A list of actors for which any post mentioning them will be dropped", + suggestions: ["actor1", "actor2"] + } + ] + } + end end diff --git a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex index 7abae37ae..e00575c2a 100644 --- a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex +++ b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do @behaviour Pleroma.Web.ActivityPub.MRF + @impl true def filter(%{"type" => "Create", "object" => child_object} = object) do scrub_policy = Pleroma.Config.get([:mrf_normalize_markup, :scrub_policy]) @@ -22,5 +23,23 @@ def filter(%{"type" => "Create", "object" => child_object} = object) do def filter(object), do: {:ok, object} + @impl true def describe, do: {:ok, %{}} + + @impl true + def config_description do + %{ + key: :mrf_normalize_markup, + related_policy: "Pleroma.Web.ActivityPub.MRF.NormalizeMarkup", + label: "MRF Normalize Markup", + description: "MRF NormalizeMarkup settings. Scrub configured hypertext markup.", + children: [ + %{ + key: :scrub_policy, + type: :module, + suggestions: [Pleroma.HTML.Scrubber.Default] + } + ] + } + end end diff --git a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex index d45d2d7e3..eb0481f20 100644 --- a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex @@ -106,4 +106,32 @@ def describe do {:ok, %{mrf_object_age: mrf_object_age}} end + + @impl true + def config_description do + %{ + key: :mrf_object_age, + related_policy: "Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy", + label: "MRF Object Age", + description: + "Rejects or delists posts based on their timestamp deviance from your server's clock.", + children: [ + %{ + key: :threshold, + type: :integer, + description: "Required age (in seconds) of a post before actions are taken.", + suggestions: [172_800] + }, + %{ + key: :actions, + type: {:list, :atom}, + description: + "A list of actions to apply to the post. `:delist` removes the post from public timelines; " <> + "`:strip_followers` removes followers from the ActivityPub recipient list ensuring they won't be delivered to home timelines; " <> + "`:reject` rejects the message entirely", + suggestions: [:delist, :strip_followers, :reject] + } + ] + } + end end diff --git a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex index 0b9ed2224..cd7665e31 100644 --- a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex +++ b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex @@ -48,4 +48,27 @@ def filter(object), do: {:ok, object} @impl true def describe, do: {:ok, %{mrf_rejectnonpublic: Config.get(:mrf_rejectnonpublic) |> Enum.into(%{})}} + + @impl true + def config_description do + %{ + key: :mrf_rejectnonpublic, + related_policy: "Pleroma.Web.ActivityPub.MRF.RejectNonPublic", + description: "RejectNonPublic drops posts with non-public visibility settings.", + label: "MRF Reject Non Public", + children: [ + %{ + key: :allow_followersonly, + label: "Allow followers-only", + type: :boolean, + description: "Whether to allow followers-only posts" + }, + %{ + key: :allow_direct, + type: :boolean, + description: "Whether to allow direct messages" + } + ] + } + end end diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 161177727..6cd91826d 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -244,4 +244,78 @@ def describe do {:ok, %{mrf_simple: mrf_simple}} end + + @impl true + def config_description do + %{ + key: :mrf_simple, + related_policy: "Pleroma.Web.ActivityPub.MRF.SimplePolicy", + label: "MRF Simple", + description: "Simple ingress policies", + children: [ + %{ + key: :media_removal, + type: {:list, :string}, + description: "List of instances to strip media attachments from", + suggestions: ["example.com", "*.example.com"] + }, + %{ + key: :media_nsfw, + label: "Media NSFW", + type: {:list, :string}, + description: "List of instances to tag all media as NSFW (sensitive) from", + suggestions: ["example.com", "*.example.com"] + }, + %{ + key: :federated_timeline_removal, + type: {:list, :string}, + description: + "List of instances to remove from the Federated (aka The Whole Known Network) Timeline", + suggestions: ["example.com", "*.example.com"] + }, + %{ + key: :reject, + type: {:list, :string}, + description: "List of instances to reject activities from (except deletes)", + suggestions: ["example.com", "*.example.com"] + }, + %{ + key: :accept, + type: {:list, :string}, + description: "List of instances to only accept activities from (except deletes)", + suggestions: ["example.com", "*.example.com"] + }, + %{ + key: :followers_only, + type: {:list, :string}, + description: "Force posts from the given instances to be visible by followers only", + suggestions: ["example.com", "*.example.com"] + }, + %{ + key: :report_removal, + type: {:list, :string}, + description: "List of instances to reject reports from", + suggestions: ["example.com", "*.example.com"] + }, + %{ + key: :avatar_removal, + type: {:list, :string}, + description: "List of instances to strip avatars from", + suggestions: ["example.com", "*.example.com"] + }, + %{ + key: :banner_removal, + type: {:list, :string}, + description: "List of instances to strip banners from", + suggestions: ["example.com", "*.example.com"] + }, + %{ + key: :reject_deletes, + type: {:list, :string}, + description: "List of instances to reject deletions from", + suggestions: ["example.com", "*.example.com"] + } + ] + } + end end diff --git a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex index 048052da6..2ec45260a 100644 --- a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex @@ -39,4 +39,28 @@ def filter(message), do: {:ok, message} @impl true def describe, do: {:ok, %{}} + + @impl true + def config_description do + %{ + key: :mrf_subchain, + related_policy: "Pleroma.Web.ActivityPub.MRF.SubchainPolicy", + label: "MRF Subchain", + description: + "This policy processes messages through an alternate pipeline when a given message matches certain criteria." <> + " All criteria are configured as a map of regular expressions to lists of policy modules.", + children: [ + %{ + key: :match_actor, + type: {:map, {:list, :string}}, + description: "Matches a series of regular expressions against the actor field", + suggestions: [ + %{ + ~r/https:\/\/example.com/s => [Pleroma.Web.ActivityPub.MRF.DropPolicy] + } + ] + } + ] + } + end end diff --git a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex index 1a28f2ba2..885bcca6f 100644 --- a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex @@ -41,4 +41,18 @@ def describe do {:ok, %{mrf_user_allowlist: mrf_user_allowlist}} end + + @impl true + def config_description do + %{ + key: :mrf_user_allowlist, + related_policy: "Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy", + description: + "The keys in this section are the domain names that the policy should apply to." <> + " Each key should be assigned a list of users that should be allowed through by their ActivityPub ID", + suggestions: [ + %{"example.org" => ["https://example.org/users/admin"]} + ] + } + end end diff --git a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex index a6c545570..f325cb680 100644 --- a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do @behaviour Pleroma.Web.ActivityPub.MRF + @impl true def filter(%{"type" => "Undo", "object" => child_message} = message) do with {:ok, _} <- filter(child_message) do {:ok, message} @@ -36,6 +37,33 @@ def filter(%{"type" => message_type} = message) do def filter(message), do: {:ok, message} + @impl true def describe, do: {:ok, %{mrf_vocabulary: Pleroma.Config.get(:mrf_vocabulary) |> Enum.into(%{})}} + + @impl true + def config_description do + %{ + key: :mrf_vocabulary, + related_policy: "Pleroma.Web.ActivityPub.MRF.VocabularyPolicy", + label: "MRF Vocabulary", + description: "Filter messages which belong to certain activity vocabularies", + children: [ + %{ + key: :accept, + type: {:list, :string}, + description: + "A list of ActivityStreams terms to accept. If empty, all supported messages are accepted.", + suggestions: ["Create", "Follow", "Mention", "Announce", "Like"] + }, + %{ + key: :reject, + type: {:list, :string}, + description: + "A list of ActivityStreams terms to reject. If empty, no messages are rejected.", + suggestions: ["Create", "Follow", "Mention", "Announce", "Like"] + } + ] + } + end end diff --git a/test/fixtures/modules/good_mrf.ex b/test/fixtures/modules/good_mrf.ex new file mode 100644 index 000000000..39d0f14ec --- /dev/null +++ b/test/fixtures/modules/good_mrf.ex @@ -0,0 +1,19 @@ +defmodule Fixtures.Modules.GoodMRF do + @behaviour Pleroma.Web.ActivityPub.MRF + + @impl true + def filter(a), do: {:ok, a} + + @impl true + def describe, do: %{} + + @impl true + def config_description do + %{ + key: :good_mrf, + related_policy: "Fixtures.Modules.GoodMRF", + label: "Good MRF", + description: "Some description" + } + end +end diff --git a/test/pleroma/web/activity_pub/mrf_test.exs b/test/pleroma/web/activity_pub/mrf_test.exs index e8cdde2e1..17aef2e16 100644 --- a/test/pleroma/web/activity_pub/mrf_test.exs +++ b/test/pleroma/web/activity_pub/mrf_test.exs @@ -87,4 +87,21 @@ test "it works as expected with mock policy" do {:ok, ^expected} = MRF.describe() end end + + test "config_descriptions/0" do + descriptions = MRF.config_descriptions() + + good_mrf = Enum.find(descriptions, fn %{key: key} -> key == :good_mrf end) + + assert good_mrf == %{ + key: :good_mrf, + related_policy: "Fixtures.Modules.GoodMRF", + label: "Good MRF", + description: "Some description", + children: [], + group: :pleroma, + tab: :mrf, + type: :group + } + end end From 2933658446f4172bb77b4d294d032fa4e2d93237 Mon Sep 17 00:00:00 2001 From: feld <feld@feld.me> Date: Tue, 10 Nov 2020 16:44:00 +0000 Subject: [PATCH 56/74] Apply 1 suggestion(s) to 1 file(s) --- docs/configuration/optimizing_beam.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md index 545392f8f..1753620c9 100644 --- a/docs/configuration/optimizing_beam.md +++ b/docs/configuration/optimizing_beam.md @@ -27,7 +27,7 @@ Check your OS documentation to adopt a similar strategy on other platforms. ### Virtual Machine and/or few CPU cores -Disable the busy-waiting +Disable the busy-waiting. This should generally only be done if you're on a platform that does burst scheduling, like AWS. **vm.args:** From 952a8c213e37b38c48890813f80e8792cb0cebe2 Mon Sep 17 00:00:00 2001 From: feld <feld@feld.me> Date: Tue, 10 Nov 2020 16:44:08 +0000 Subject: [PATCH 57/74] Apply 1 suggestion(s) to 1 file(s) --- docs/configuration/optimizing_beam.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md index 1753620c9..ecfae86d9 100644 --- a/docs/configuration/optimizing_beam.md +++ b/docs/configuration/optimizing_beam.md @@ -39,7 +39,7 @@ Disable the busy-waiting. This should generally only be done if you're on a plat ### Dedicated Hardware -Enable more busy waiting, increase the internal maximum limit of BEAM processes and ports +Enable more busy waiting, increase the internal maximum limit of BEAM processes and ports. You can use this if you run on dedicated hardware, but it is not necessary. **vm.args:** From 776067a9a32fd2b536c6782a0aa7c2dfd8444280 Mon Sep 17 00:00:00 2001 From: feld <feld@feld.me> Date: Tue, 10 Nov 2020 16:44:17 +0000 Subject: [PATCH 58/74] Apply 1 suggestion(s) to 1 file(s) --- docs/configuration/optimizing_beam.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md index ecfae86d9..a79d188ca 100644 --- a/docs/configuration/optimizing_beam.md +++ b/docs/configuration/optimizing_beam.md @@ -6,6 +6,8 @@ This strategy is very successful in making a performant and responsive applicati More adventurous admins can be creative with CPU affinity (e.g., *taskset* for Linux and *cpuset* on FreeBSD) to pin processes to specific CPUs and eliminate much of this contention. The most important advice is to run as few processes as possible on your server to achieve the best performance. Even idle background processes can occasionally create [software interrupts](https://en.wikipedia.org/wiki/Interrupt) and take attention away from the executing process creating latency spikes and invalidation of the CPU caches as they must be cleared when switching between processes for security. +Please only change these settings if you are experiencing issues or really know what you are doing. In general, there's no need to change these settings. + ## VPS Provider Recommendations ### Good From 7681b4c5cd683bcc8c91b1d296261e7e2b11fd88 Mon Sep 17 00:00:00 2001 From: feld <feld@feld.me> Date: Tue, 10 Nov 2020 16:44:23 +0000 Subject: [PATCH 59/74] Apply 1 suggestion(s) to 1 file(s) --- docs/configuration/optimizing_beam.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/optimizing_beam.md b/docs/configuration/optimizing_beam.md index a79d188ca..e336bd36c 100644 --- a/docs/configuration/optimizing_beam.md +++ b/docs/configuration/optimizing_beam.md @@ -12,7 +12,7 @@ Please only change these settings if you are experiencing issues or really know ### Good -* ???? +* Hetzner Cloud ### Bad From efc27f64643f24e97c604efa02c15bfe3210cf3b Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Wed, 11 Nov 2020 10:10:57 +0300 Subject: [PATCH 60/74] fix for adminFE - revert UserAllowPolicy description - MRF descriptions order --- config/description.exs | 34 --------------- lib/pleroma/web/activity_pub/mrf.ex | 42 +++++++++++++++++-- .../mrf/user_allow_list_policy.ex | 32 ++++++++------ 3 files changed, 58 insertions(+), 50 deletions(-) diff --git a/config/description.exs b/config/description.exs index 69072312c..0552b37e0 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1554,40 +1554,6 @@ } ] }, - %{ - group: :pleroma, - key: :mrf, - tab: :mrf, - label: "MRF", - type: :group, - description: "General MRF settings", - children: [ - %{ - key: :policies, - type: [:module, {:list, :module}], - description: - "A list of MRF policies enabled. Module names are shortened (removed leading `Pleroma.Web.ActivityPub.MRF.` part), but on adding custom module you need to use full name.", - suggestions: {:list_behaviour_implementations, Pleroma.Web.ActivityPub.MRF} - }, - %{ - key: :transparency, - label: "MRF transparency", - type: :boolean, - description: - "Make the content of your Message Rewrite Facility settings public (via nodeinfo)" - }, - %{ - key: :transparency_exclusions, - label: "MRF transparency exclusions", - type: {:list, :string}, - description: - "Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.", - suggestions: [ - "exclusion.com" - ] - } - ] - }, %{ group: :pleroma, key: :media_proxy, diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 656e4c7ca..ce125a696 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -5,10 +5,46 @@ defmodule Pleroma.Web.ActivityPub.MRF do require Logger + @mrf_config_descriptions [ + %{ + group: :pleroma, + key: :mrf, + tab: :mrf, + label: "MRF", + type: :group, + description: "General MRF settings", + children: [ + %{ + key: :policies, + type: [:module, {:list, :module}], + description: + "A list of MRF policies enabled. Module names are shortened (removed leading `Pleroma.Web.ActivityPub.MRF.` part), but on adding custom module you need to use full name.", + suggestions: {:list_behaviour_implementations, Pleroma.Web.ActivityPub.MRF} + }, + %{ + key: :transparency, + label: "MRF transparency", + type: :boolean, + description: + "Make the content of your Message Rewrite Facility settings public (via nodeinfo)" + }, + %{ + key: :transparency_exclusions, + label: "MRF transparency exclusions", + type: {:list, :string}, + description: + "Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.", + suggestions: [ + "exclusion.com" + ] + } + ] + } + ] + @default_description %{ label: "", - description: "", - children: [] + description: "" } @required_description_keys [:key, :related_policy] @@ -107,7 +143,7 @@ def config_descriptions do end def config_descriptions(policies) do - Enum.reduce(policies, [], fn policy, acc -> + Enum.reduce(policies, @mrf_config_descriptions, fn policy, acc -> if function_exported?(policy, :config_description, 0) do description = @default_description diff --git a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex index 885bcca6f..f2859abde 100644 --- a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex @@ -42,17 +42,23 @@ def describe do {:ok, %{mrf_user_allowlist: mrf_user_allowlist}} end - @impl true - def config_description do - %{ - key: :mrf_user_allowlist, - related_policy: "Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy", - description: - "The keys in this section are the domain names that the policy should apply to." <> - " Each key should be assigned a list of users that should be allowed through by their ActivityPub ID", - suggestions: [ - %{"example.org" => ["https://example.org/users/admin"]} - ] - } - end + # TODO: change way of getting settings on `lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex:18` to use `hosts` subkey + # @impl true + # def config_description do + # %{ + # key: :mrf_user_allowlist, + # related_policy: "Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy", + # description: "Accept-list of users from specified instances", + # children: [ + # %{ + # key: :hosts, + # type: :map, + # description: + # "The keys in this section are the domain names that the policy should apply to." <> + # " Each key should be assigned a list of users that should be allowed through by their ActivityPub ID", + # suggestions: [%{"example.org" => ["https://example.org/users/admin"]}] + # } + # ] + # } + # end end From f97f24b067b6d0205f093b04aeb08c3d56faa7b4 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Wed, 11 Nov 2020 10:48:03 +0300 Subject: [PATCH 61/74] making credo happy and test fix --- lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex | 3 ++- test/pleroma/web/activity_pub/mrf_test.exs | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex index f2859abde..e9d0d0503 100644 --- a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex @@ -55,7 +55,8 @@ def describe do # type: :map, # description: # "The keys in this section are the domain names that the policy should apply to." <> - # " Each key should be assigned a list of users that should be allowed through by their ActivityPub ID", + # " Each key should be assigned a list of users that should be allowed " <> + # "through by their ActivityPub ID", # suggestions: [%{"example.org" => ["https://example.org/users/admin"]}] # } # ] diff --git a/test/pleroma/web/activity_pub/mrf_test.exs b/test/pleroma/web/activity_pub/mrf_test.exs index 17aef2e16..44a9cf086 100644 --- a/test/pleroma/web/activity_pub/mrf_test.exs +++ b/test/pleroma/web/activity_pub/mrf_test.exs @@ -98,7 +98,6 @@ test "config_descriptions/0" do related_policy: "Fixtures.Modules.GoodMRF", label: "Good MRF", description: "Some description", - children: [], group: :pleroma, tab: :mrf, type: :group From 8da9f919f82ac45c4519910a7e24cac2b797061f Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Wed, 11 Nov 2020 18:49:15 +0300 Subject: [PATCH 62/74] little changes for MRF config descriptions - log level reduction, if policy doesn't implement config_description method - docs in dev.md --- docs/dev.md | 23 +++++++++++++++++++++++ lib/pleroma/web/activity_pub/mrf.ex | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/dev.md b/docs/dev.md index 22e0691f1..aa89a941f 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -21,3 +21,26 @@ This document contains notes and guidelines for Pleroma developers. ## Auth-related configuration, OAuth consumer mode etc. See `Authentication` section of [the configuration cheatsheet](configuration/cheatsheet.md#authentication). + +## MRF policies descriptions + +If MRF policy depends on config, it can be added into MRF tab to adminFE by adding `config_description/0` method, which returns map with special structure. + +Example: + +```elixir +%{ + key: :mrf_activity_expiration, + related_policy: "Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy", + label: "MRF Activity Expiration Policy", + description: "Adds automatic expiration to all local activities", + children: [ + %{ + key: :days, + type: :integer, + description: "Default global expiration time for all local activities (in days)", + suggestions: [90, 365] + } + ] + } +``` diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index ce125a696..6e73b2f22 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -164,7 +164,7 @@ def config_descriptions(policies) do acc end else - Logger.info( + Logger.debug( "#{policy} is excluded from config descriptions, because does not implement `config_description/0` method." ) From 631def2df228ceb0ec8921c63b90867758e0c308 Mon Sep 17 00:00:00 2001 From: lain <lain@soykaf.club> Date: Wed, 11 Nov 2020 17:10:59 +0100 Subject: [PATCH 63/74] RedirectController: Don't replace title, but inject into the meta --- lib/pleroma/web/fallback/redirect_controller.ex | 6 ++---- test/pleroma/web/fallback_test.exs | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/fallback/redirect_controller.ex b/lib/pleroma/web/fallback/redirect_controller.ex index 712991c18..1ac1319f8 100644 --- a/lib/pleroma/web/fallback/redirect_controller.ex +++ b/lib/pleroma/web/fallback/redirect_controller.ex @@ -41,8 +41,7 @@ def redirector_with_meta(conn, params) do response = index_content - |> String.replace(~r/<title>.+?<\/title>/, title) - |> String.replace("<!--server-generated-meta-->", tags <> preloads) + |> String.replace("<!--server-generated-meta-->", tags <> preloads <> title) conn |> put_resp_content_type("text/html") @@ -60,8 +59,7 @@ def redirector_with_preload(conn, params) do response = index_content - |> String.replace(~r/<title>.+?<\/title>/, title) - |> String.replace("<!--server-generated-meta-->", preloads) + |> String.replace("<!--server-generated-meta-->", preloads <> title) conn |> put_resp_content_type("text/html") diff --git a/test/pleroma/web/fallback_test.exs b/test/pleroma/web/fallback_test.exs index a65865860..46c7bad1c 100644 --- a/test/pleroma/web/fallback_test.exs +++ b/test/pleroma/web/fallback_test.exs @@ -20,15 +20,26 @@ test "GET /*path", %{conn: conn} do end end + test "GET /*path adds a title", %{conn: conn} do + clear_config([:instance, :name], "a cool title") + + assert conn + |> get("/") + |> html_response(200) =~ "<title>a cool title</title>" + end + describe "preloaded data and metadata attached to" do test "GET /:maybe_nickname_or_id", %{conn: conn} do + clear_config([:instance, :name], "a cool title") + user = insert(:user) user_missing = get(conn, "/foo") user_present = get(conn, "/#{user.nickname}") - assert(html_response(user_missing, 200) =~ "<!--server-generated-meta-->") + assert html_response(user_missing, 200) =~ "<!--server-generated-meta-->" refute html_response(user_present, 200) =~ "<!--server-generated-meta-->" assert html_response(user_present, 200) =~ "initial-results" + assert html_response(user_present, 200) =~ "<title>a cool title</title>" end test "GET /*path", %{conn: conn} do @@ -44,10 +55,13 @@ test "GET /*path", %{conn: conn} do describe "preloaded data is attached to" do test "GET /main/public", %{conn: conn} do + clear_config([:instance, :name], "a cool title") + public_page = get(conn, "/main/public") refute html_response(public_page, 200) =~ "<!--server-generated-meta-->" assert html_response(public_page, 200) =~ "initial-results" + assert html_response(public_page, 200) =~ "<title>a cool title</title>" end test "GET /main/all", %{conn: conn} do From 25bd64b03ac78857dc4d560be0c75ea096080c33 Mon Sep 17 00:00:00 2001 From: lain <lain@soykaf.club> Date: Wed, 11 Nov 2020 17:17:41 +0100 Subject: [PATCH 64/74] Bundled FE: Remove title tag --- priv/static/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priv/static/index.html b/priv/static/index.html index f5690a8d6..e848c5f8c 100644 --- a/priv/static/index.html +++ b/priv/static/index.html @@ -1 +1 @@ -<!DOCTYPE html><html lang=en><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,user-scalable=no"><title>Pleroma</title><!--server-generated-meta--><link rel=icon type=image/png href=/favicon.png><link href=/static/css/app.77b1644622e3bae24b6b.css rel=stylesheet><link href=/static/fontello.1600365488745.css rel=stylesheet></head><body class=hidden><noscript>To use Pleroma, please enable JavaScript.</noscript><div id=app></div><script type=text/javascript src=/static/js/vendors~app.90c4af83c1ae68f4cd95.js></script><script type=text/javascript src=/static/js/app.826c44232e0a76bbd9ba.js></script></body></html> \ No newline at end of file +<!DOCTYPE html><html lang=en><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,user-scalable=no"><!--server-generated-meta--><link rel=icon type=image/png href=/favicon.png><link href=/static/css/app.77b1644622e3bae24b6b.css rel=stylesheet><link href=/static/fontello.1600365488745.css rel=stylesheet></head><body class=hidden><noscript>To use Pleroma, please enable JavaScript.</noscript><div id=app></div><script type=text/javascript src=/static/js/vendors~app.90c4af83c1ae68f4cd95.js></script><script type=text/javascript src=/static/js/app.826c44232e0a76bbd9ba.js></script></body></html> From b0e4e0cf2a151900d29da56424f596f3defa23e3 Mon Sep 17 00:00:00 2001 From: lain <lain@soykaf.club> Date: Wed, 11 Nov 2020 17:19:09 +0100 Subject: [PATCH 65/74] Changelog: Add info about title injection --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b15ddb943..b619bd891 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Account backup - Configuration: Add `:instance, autofollowing_nicknames` setting to provide a way to make accounts automatically follow new users that register on the local Pleroma instance. - Ability to view remote timelines, with ex. `/api/v1/timelines/public?instance=lain.com` and streams `public:remote` and `public:remote:media` +- The site title is now injected as a `title` tag like preloads or metadata. ### Changed From 6fd72e9e8526680836e1bf34c58e10b66dcfee8c Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@FreeBSD.org> Date: Wed, 11 Nov 2020 12:27:51 -0600 Subject: [PATCH 66/74] Ingest blurhash for attachments if they were federated --- .../activity_pub/object_validators/attachment_validator.ex | 3 ++- .../object_validators/attachment_validator_test.exs | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index df102a134..f96fd54bf 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do field(:type, :string) field(:mediaType, :string, default: "application/octet-stream") field(:name, :string) + field(:blurhash, :string) embeds_many :url, UrlObjectValidator, primary_key: false do field(:type, :string) @@ -41,7 +42,7 @@ def changeset(struct, data) do |> fix_url() struct - |> cast(data, [:type, :mediaType, :name]) + |> cast(data, [:type, :mediaType, :name, :blurhash]) |> cast_embed(:url, with: &url_changeset/2) |> validate_inclusion(:type, ~w[Link Document Audio Image Video]) |> validate_required([:type, :mediaType, :url]) diff --git a/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs index 760388e80..2e1975a79 100644 --- a/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs @@ -33,7 +33,8 @@ test "it turns mastodon attachments into our attachments" do "http://mastodon.example.org/system/media_attachments/files/000/000/002/original/334ce029e7bfb920.jpg", "type" => "Document", "name" => nil, - "mediaType" => "image/jpeg" + "mediaType" => "image/jpeg", + "blurhash" => "UD9jJz~VSbR#xT$~%KtQX9R,WAs9RjWBs:of" } {:ok, attachment} = @@ -50,6 +51,7 @@ test "it turns mastodon attachments into our attachments" do ] = attachment.url assert attachment.mediaType == "image/jpeg" + assert attachment.blurhash == "UD9jJz~VSbR#xT$~%KtQX9R,WAs9RjWBs:of" end test "it handles our own uploads" do From 2254e5e5958803beef94e2d01bdb04647a1f82c9 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@FreeBSD.org> Date: Wed, 11 Nov 2020 12:51:13 -0600 Subject: [PATCH 67/74] Render blurhashes in Mastodon API --- lib/pleroma/web/mastodon_api/views/status_view.ex | 3 ++- test/pleroma/web/mastodon_api/views/status_view_test.exs | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 435bcde15..7cbbd3750 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -435,7 +435,8 @@ def render("attachment.json", %{attachment: attachment}) do text_url: href, type: type, description: attachment["name"], - pleroma: %{mime_type: media_type} + pleroma: %{mime_type: media_type}, + blurhash: attachment["blurhash"] } end diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index 70d829979..665199f97 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -420,6 +420,7 @@ test "attachments" do "href" => "someurl" } ], + "blurhash" => "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn", "uuid" => 6 } @@ -431,7 +432,8 @@ test "attachments" do preview_url: "someurl", text_url: "someurl", description: nil, - pleroma: %{mime_type: "image/png"} + pleroma: %{mime_type: "image/png"}, + blurhash: "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn" } api_spec = Pleroma.Web.ApiSpec.spec() From 2156de2feef373eecdf8e16f21ac7c4b1ee99995 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@FreeBSD.org> Date: Wed, 11 Nov 2020 13:39:02 -0600 Subject: [PATCH 68/74] Ingest blurhash field during transmogrification --- lib/pleroma/web/activity_pub/transmogrifier.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 39c8f7e39..0bcd1db22 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -252,6 +252,7 @@ def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachm } |> Maps.put_if_present("mediaType", media_type) |> Maps.put_if_present("name", data["name"]) + |> Maps.put_if_present("blurhash", data["blurhash"]) else nil end From 966663c3f89519ad0e3b434e611dcceed6f99822 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@FreeBSD.org> Date: Wed, 11 Nov 2020 16:17:35 -0600 Subject: [PATCH 69/74] Fix tests for other attachment types --- .../web/activity_pub/transmogrifier/audio_handling_test.exs | 1 + .../web/activity_pub/transmogrifier/video_handling_test.exs | 2 ++ 2 files changed, 3 insertions(+) diff --git a/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs index 0636d00c5..eef7ab4c6 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs @@ -69,6 +69,7 @@ test "Funkwhale Audio object" do "mediaType" => "audio/ogg", "type" => "Link", "name" => nil, + "blurhash" => nil, "url" => [ %{ "href" => diff --git a/test/pleroma/web/activity_pub/transmogrifier/video_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/video_handling_test.exs index 69c953a2e..57411fafa 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/video_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/video_handling_test.exs @@ -54,6 +54,7 @@ test "it remaps video URLs as attachments if necessary" do "type" => "Link", "mediaType" => "video/mp4", "name" => nil, + "blurhash" => nil, "url" => [ %{ "href" => @@ -76,6 +77,7 @@ test "it remaps video URLs as attachments if necessary" do "type" => "Link", "mediaType" => "video/mp4", "name" => nil, + "blurhash" => nil, "url" => [ %{ "href" => From 6ca709816f74f1171423c7bc040619fca57a2087 Mon Sep 17 00:00:00 2001 From: rinpatch <rinpatch@sdf.org> Date: Wed, 28 Oct 2020 18:08:23 +0300 Subject: [PATCH 70/74] Fix object spoofing vulnerability in attachments Validate the content-type of the response when fetching an object, according to https://www.w3.org/TR/activitypub/#x3-2-retrieving-objects. content-type headers had to be added to many mocks in order to support this, some of this was done with a regex. While I did go over the resulting files to check I didn't modify anything unrelated, there is a possibility I missed something. Closes pleroma#1948 --- lib/pleroma/object/fetcher.ex | 20 +- test/fixtures/spoofed-object.json | 26 +++ test/pleroma/object/fetcher_test.exs | 27 ++- test/pleroma/object_test.exs | 15 +- .../web/activity_pub/activity_pub_test.exs | 24 ++- .../transmogrifier/announce_handling_test.exs | 6 +- .../transmogrifier/article_handling_test.exs | 15 +- .../transmogrifier/audio_handling_test.exs | 3 +- .../transmogrifier/event_handling_test.exs | 6 +- test/support/http_request_mock.ex | 190 +++++++++++++----- 10 files changed, 253 insertions(+), 79 deletions(-) create mode 100644 test/fixtures/spoofed-object.json diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 169298b34..ae4301738 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -232,8 +232,24 @@ defp get_object_http(id) do |> sign_fetch(id, date) case HTTP.get(id, headers) do - {:ok, %{body: body, status: code}} when code in 200..299 -> - {:ok, body} + {:ok, %{body: body, status: code, headers: headers}} when code in 200..299 -> + case List.keyfind(headers, "content-type", 0) do + {_, content_type} -> + case Plug.Conn.Utils.media_type(content_type) do + {:ok, "application", "activity+json", _} -> + {:ok, body} + + {:ok, "application", "ld+json", + %{"profile" => "https://www.w3.org/ns/activitystreams"}} -> + {:ok, body} + + _ -> + {:error, {:content_type, content_type}} + end + + _ -> + {:error, {:content_type, nil}} + end {:ok, %{status: code}} when code in [404, 410] -> {:error, "Object has been deleted"} diff --git a/test/fixtures/spoofed-object.json b/test/fixtures/spoofed-object.json new file mode 100644 index 000000000..91e34307d --- /dev/null +++ b/test/fixtures/spoofed-object.json @@ -0,0 +1,26 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://patch.cx/schemas/litepub-0.1.jsonld", + { + "@language": "und" + } + ], + "actor": "https://patch.cx/users/rin", + "attachment": [], + "attributedTo": "https://patch.cx/users/rin", + "cc": [ + "https://patch.cx/users/rin/followers" + ], + "content": "Oracle Corporation (NYSE: ORCL) today announced that it has signed a definitive merger agreement to acquire Pleroma AG (FRA: PLA), for $26.50 per share (approximately $10.3 billion). The transaction has been approved by the boards of directors of both companies and should close by early January.", + "context": "https://patch.cx/contexts/spoof", + "id": "https://patch.cx/objects/spoof", + "published": "2020-10-23T18:02:06.038856Z", + "sensitive": false, + "summary": "Oracle buys Pleroma", + "tag": [], + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "type": "Note" +} diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs index 14d2c645f..7df6af7fe 100644 --- a/test/pleroma/object/fetcher_test.exs +++ b/test/pleroma/object/fetcher_test.exs @@ -21,6 +21,17 @@ defmodule Pleroma.Object.FetcherTest do %{method: :get, url: "https://mastodon.example.org/users/userisgone404"} -> %Tesla.Env{status: 404} + %{ + method: :get, + url: + "https://patch.cx/media/03ca3c8b4ac3ddd08bf0f84be7885f2f88de0f709112131a22d83650819e36c2.json" + } -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/json"}], + body: File.read!("test/fixtures/spoofed-object.json") + } + env -> apply(HttpRequestMock, :request, [env]) end) @@ -34,19 +45,22 @@ defmodule Pleroma.Object.FetcherTest do %{method: :get, url: "https://social.sakamoto.gq/notice/9wTkLEnuq47B25EehM"} -> %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/fetch_mocks/9wTkLEnuq47B25EehM.json") + body: File.read!("test/fixtures/fetch_mocks/9wTkLEnuq47B25EehM.json"), + headers: HttpRequestMock.activitypub_object_headers() } %{method: :get, url: "https://social.sakamoto.gq/users/eal"} -> %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/fetch_mocks/eal.json") + body: File.read!("test/fixtures/fetch_mocks/eal.json"), + headers: HttpRequestMock.activitypub_object_headers() } %{method: :get, url: "https://busshi.moe/users/tuxcrafting/statuses/104410921027210069"} -> %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/fetch_mocks/104410921027210069.json") + body: File.read!("test/fixtures/fetch_mocks/104410921027210069.json"), + headers: HttpRequestMock.activitypub_object_headers() } %{method: :get, url: "https://busshi.moe/users/tuxcrafting"} -> @@ -132,6 +146,13 @@ test "Return MRF reason when fetched status is rejected by one" do "http://mastodon.example.org/@admin/99541947525187367" ) end + + test "it does not fetch a spoofed object uploaded on an instance as an attachment" do + assert {:error, _} = + Fetcher.fetch_object_from_id( + "https://patch.cx/media/03ca3c8b4ac3ddd08bf0f84be7885f2f88de0f709112131a22d83650819e36c2.json" + ) + end end describe "implementation quirks" do diff --git a/test/pleroma/object_test.exs b/test/pleroma/object_test.exs index 99caba336..5d4e6fb84 100644 --- a/test/pleroma/object_test.exs +++ b/test/pleroma/object_test.exs @@ -281,7 +281,11 @@ test "does not fetch unknown objects when fetch_remote is false" do setup do mock(fn %{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} -> - %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/poll_original.json")} + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/poll_original.json"), + headers: HttpRequestMock.activitypub_object_headers() + } env -> apply(HttpRequestMock, :request, [env]) @@ -315,7 +319,8 @@ test "refetches if the time since the last refetch is greater than the interval" mock_modified.(%Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/poll_modified.json") + body: File.read!("test/fixtures/tesla_mock/poll_modified.json"), + headers: HttpRequestMock.activitypub_object_headers() }) updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1) @@ -359,7 +364,8 @@ test "does not refetch if the time since the last refetch is greater than the in mock_modified.(%Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/poll_modified.json") + body: File.read!("test/fixtures/tesla_mock/poll_modified.json"), + headers: HttpRequestMock.activitypub_object_headers() }) updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: 100) @@ -387,7 +393,8 @@ test "preserves internal fields on refetch", %{mock_modified: mock_modified} do mock_modified.(%Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/poll_modified.json") + body: File.read!("test/fixtures/tesla_mock/poll_modified.json"), + headers: HttpRequestMock.activitypub_object_headers() }) updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1) diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index 43bd14ee6..3eeb0f735 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -1426,19 +1426,25 @@ test "doesn't crash when follower and following counters are hidden" do mock(fn env -> case env.url do "http://localhost:4001/users/masto_hidden_counters/following" -> - json(%{ - "@context" => "https://www.w3.org/ns/activitystreams", - "id" => "http://localhost:4001/users/masto_hidden_counters/followers" - }) + json( + %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "id" => "http://localhost:4001/users/masto_hidden_counters/followers" + }, + headers: HttpRequestMock.activitypub_object_headers() + ) "http://localhost:4001/users/masto_hidden_counters/following?page=1" -> %Tesla.Env{status: 403, body: ""} "http://localhost:4001/users/masto_hidden_counters/followers" -> - json(%{ - "@context" => "https://www.w3.org/ns/activitystreams", - "id" => "http://localhost:4001/users/masto_hidden_counters/following" - }) + json( + %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "id" => "http://localhost:4001/users/masto_hidden_counters/following" + }, + headers: HttpRequestMock.activitypub_object_headers() + ) "http://localhost:4001/users/masto_hidden_counters/followers?page=1" -> %Tesla.Env{status: 403, body: ""} @@ -2278,7 +2284,7 @@ test "allow fetching of accounts with an empty string name field" do Tesla.Mock.mock(fn %{method: :get, url: "https://princess.cat/users/mewmew"} -> file = File.read!("test/fixtures/mewmew_no_name.json") - %Tesla.Env{status: 200, body: file} + %Tesla.Env{status: 200, body: file, headers: HttpRequestMock.activitypub_object_headers()} end) {:ok, user} = ActivityPub.make_user_from_ap_id("https://princess.cat/users/mewmew") diff --git a/test/pleroma/web/activity_pub/transmogrifier/announce_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/announce_handling_test.exs index 54335acdb..99c296c74 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/announce_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/announce_handling_test.exs @@ -60,7 +60,11 @@ test "it works for incoming announces, fetching the announced object" do Tesla.Mock.mock(fn %{method: :get} -> - %Tesla.Env{status: 200, body: File.read!("test/fixtures/mastodon-note-object.json")} + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/mastodon-note-object.json"), + headers: HttpRequestMock.activitypub_object_headers() + } end) _user = insert(:user, local: false, ap_id: data["actor"]) diff --git a/test/pleroma/web/activity_pub/transmogrifier/article_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/article_handling_test.exs index 9b12a470a..b0ae804c5 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/article_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/article_handling_test.exs @@ -13,7 +13,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.ArticleHandlingTest do test "Pterotype (Wordpress Plugin) Article" do Tesla.Mock.mock(fn %{url: "https://wedistribute.org/wp-json/pterotype/v1/actor/-blog"} -> - %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/wedistribute-user.json")} + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/wedistribute-user.json"), + headers: HttpRequestMock.activitypub_object_headers() + } end) data = @@ -36,13 +40,15 @@ test "Plume Article" do %{url: "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"} -> %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-article.json") + body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-article.json"), + headers: HttpRequestMock.activitypub_object_headers() } %{url: "https://baptiste.gelez.xyz/@/BaptisteGelez"} -> %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-user.json") + body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-user.json"), + headers: HttpRequestMock.activitypub_object_headers() } end) @@ -61,7 +67,8 @@ test "Prismo Article" do Tesla.Mock.mock(fn %{url: "https://prismo.news/@mxb"} -> %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/https___prismo.news__mxb.json") + body: File.read!("test/fixtures/tesla_mock/https___prismo.news__mxb.json"), + headers: HttpRequestMock.activitypub_object_headers() } end) diff --git a/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs index eef7ab4c6..6eeb1c863 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs @@ -48,7 +48,8 @@ test "Funkwhale Audio object" do %{url: "https://channels.tests.funkwhale.audio/federation/actors/compositions"} -> %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/funkwhale_channel.json") + body: File.read!("test/fixtures/tesla_mock/funkwhale_channel.json"), + headers: HttpRequestMock.activitypub_object_headers() } end) diff --git a/test/pleroma/web/activity_pub/transmogrifier/event_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/event_handling_test.exs index 7f1ef2cbd..d7c55cfbe 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/event_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/event_handling_test.exs @@ -13,13 +13,15 @@ test "Mobilizon Event object" do %{url: "https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39"} -> %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/mobilizon.org-event.json") + body: File.read!("test/fixtures/tesla_mock/mobilizon.org-event.json"), + headers: HttpRequestMock.activitypub_object_headers() } %{url: "https://mobilizon.org/@tcit"} -> %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/mobilizon.org-user.json") + body: File.read!("test/fixtures/tesla_mock/mobilizon.org-user.json"), + headers: HttpRequestMock.activitypub_object_headers() } end) diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index cb022333f..93464ebff 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -5,6 +5,8 @@ defmodule HttpRequestMock do require Logger + def activitypub_object_headers, do: [{"content-type", "application/activity+json"}] + def request( %Tesla.Env{ url: url, @@ -34,7 +36,8 @@ def get("https://osada.macgirvin.com/channel/mike", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/https___osada.macgirvin.com_channel_mike.json") + body: File.read!("test/fixtures/tesla_mock/https___osada.macgirvin.com_channel_mike.json"), + headers: activitypub_object_headers() }} end @@ -42,7 +45,8 @@ def get("https://shitposter.club/users/moonman", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/moonman@shitposter.club.json") + body: File.read!("test/fixtures/tesla_mock/moonman@shitposter.club.json"), + headers: activitypub_object_headers() }} end @@ -50,7 +54,8 @@ def get("https://mastodon.social/users/emelie/statuses/101849165031453009", _, _ {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/status.emelie.json") + body: File.read!("test/fixtures/tesla_mock/status.emelie.json"), + headers: activitypub_object_headers() }} end @@ -66,7 +71,8 @@ def get("https://mastodon.social/users/emelie", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/emelie.json") + body: File.read!("test/fixtures/tesla_mock/emelie.json"), + headers: activitypub_object_headers() }} end @@ -78,7 +84,8 @@ def get("https://mastodon.sdf.org/users/rinpatch", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/rinpatch.json") + body: File.read!("test/fixtures/tesla_mock/rinpatch.json"), + headers: activitypub_object_headers() }} end @@ -86,7 +93,8 @@ def get("https://patch.cx/objects/tesla_mock/poll_attachment", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/poll_attachment.json") + body: File.read!("test/fixtures/tesla_mock/poll_attachment.json"), + headers: activitypub_object_headers() }} end @@ -99,7 +107,8 @@ def get( {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/webfinger_emelie.json") + body: File.read!("test/fixtures/tesla_mock/webfinger_emelie.json"), + headers: activitypub_object_headers() }} end @@ -112,7 +121,8 @@ def get( {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/mike@osada.macgirvin.com.json") + body: File.read!("test/fixtures/tesla_mock/mike@osada.macgirvin.com.json"), + headers: activitypub_object_headers() }} end @@ -190,7 +200,8 @@ def get( {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/lucifermysticus.json") + body: File.read!("test/fixtures/tesla_mock/lucifermysticus.json"), + headers: activitypub_object_headers() }} end @@ -198,7 +209,8 @@ def get("https://prismo.news/@mxb", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/https___prismo.news__mxb.json") + body: File.read!("test/fixtures/tesla_mock/https___prismo.news__mxb.json"), + headers: activitypub_object_headers() }} end @@ -211,7 +223,8 @@ def get( {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/kaniini@hubzilla.example.org.json") + body: File.read!("test/fixtures/tesla_mock/kaniini@hubzilla.example.org.json"), + headers: activitypub_object_headers() }} end @@ -219,7 +232,8 @@ def get("https://niu.moe/users/rye", _, _, [{"accept", "application/activity+jso {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/rye.json") + body: File.read!("test/fixtures/tesla_mock/rye.json"), + headers: activitypub_object_headers() }} end @@ -227,7 +241,8 @@ def get("https://n1u.moe/users/rye", _, _, [{"accept", "application/activity+jso {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/rye.json") + body: File.read!("test/fixtures/tesla_mock/rye.json"), + headers: activitypub_object_headers() }} end @@ -246,7 +261,8 @@ def get("https://puckipedia.com/", _, _, [{"accept", "application/activity+json" {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/puckipedia.com.json") + body: File.read!("test/fixtures/tesla_mock/puckipedia.com.json"), + headers: activitypub_object_headers() }} end @@ -254,7 +270,8 @@ def get("https://peertube.moe/accounts/7even", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/7even.json") + body: File.read!("test/fixtures/tesla_mock/7even.json"), + headers: activitypub_object_headers() }} end @@ -262,7 +279,8 @@ def get("https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/peertube.moe-vid.json") + body: File.read!("test/fixtures/tesla_mock/peertube.moe-vid.json"), + headers: activitypub_object_headers() }} end @@ -270,7 +288,8 @@ def get("https://framatube.org/accounts/framasoft", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/https___framatube.org_accounts_framasoft.json") + body: File.read!("test/fixtures/tesla_mock/https___framatube.org_accounts_framasoft.json"), + headers: activitypub_object_headers() }} end @@ -278,7 +297,8 @@ def get("https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206 {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/framatube.org-video.json") + body: File.read!("test/fixtures/tesla_mock/framatube.org-video.json"), + headers: activitypub_object_headers() }} end @@ -286,7 +306,8 @@ def get("https://peertube.social/accounts/craigmaloney", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/craigmaloney.json") + body: File.read!("test/fixtures/tesla_mock/craigmaloney.json"), + headers: activitypub_object_headers() }} end @@ -294,7 +315,8 @@ def get("https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34 {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/peertube-social.json") + body: File.read!("test/fixtures/tesla_mock/peertube-social.json"), + headers: activitypub_object_headers() }} end @@ -304,7 +326,8 @@ def get("https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39", _, {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/mobilizon.org-event.json") + body: File.read!("test/fixtures/tesla_mock/mobilizon.org-event.json"), + headers: activitypub_object_headers() }} end @@ -312,7 +335,8 @@ def get("https://mobilizon.org/@tcit", _, _, [{"accept", "application/activity+j {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/mobilizon.org-user.json") + body: File.read!("test/fixtures/tesla_mock/mobilizon.org-user.json"), + headers: activitypub_object_headers() }} end @@ -320,7 +344,8 @@ def get("https://baptiste.gelez.xyz/@/BaptisteGelez", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-user.json") + body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-user.json"), + headers: activitypub_object_headers() }} end @@ -328,7 +353,8 @@ def get("https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june- {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-article.json") + body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-article.json"), + headers: activitypub_object_headers() }} end @@ -336,7 +362,8 @@ def get("https://wedistribute.org/wp-json/pterotype/v1/object/85810", _, _, _) d {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/wedistribute-article.json") + body: File.read!("test/fixtures/tesla_mock/wedistribute-article.json"), + headers: activitypub_object_headers() }} end @@ -344,7 +371,8 @@ def get("https://wedistribute.org/wp-json/pterotype/v1/actor/-blog", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/wedistribute-user.json") + body: File.read!("test/fixtures/tesla_mock/wedistribute-user.json"), + headers: activitypub_object_headers() }} end @@ -352,7 +380,8 @@ def get("http://mastodon.example.org/users/admin", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/admin@mastdon.example.org.json") + body: File.read!("test/fixtures/tesla_mock/admin@mastdon.example.org.json"), + headers: activitypub_object_headers() }} end @@ -362,7 +391,8 @@ def get("http://mastodon.example.org/users/relay", _, _, [ {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/relay@mastdon.example.org.json") + body: File.read!("test/fixtures/tesla_mock/relay@mastdon.example.org.json"), + headers: activitypub_object_headers() }} end @@ -482,7 +512,8 @@ def get( {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/pekorino@pawoo.net_host_meta.json") + body: File.read!("test/fixtures/tesla_mock/pekorino@pawoo.net_host_meta.json"), + headers: activitypub_object_headers() }} end @@ -543,7 +574,8 @@ def get( {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/mastodon-note-object.json") + body: File.read!("test/fixtures/mastodon-note-object.json"), + headers: activitypub_object_headers() }} end @@ -567,7 +599,8 @@ def get("https://mstdn.io/users/mayuutann", _, _, [{"accept", "application/activ {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/mayumayu.json") + body: File.read!("test/fixtures/tesla_mock/mayumayu.json"), + headers: activitypub_object_headers() }} end @@ -580,7 +613,8 @@ def get( {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/mayumayupost.json") + body: File.read!("test/fixtures/tesla_mock/mayumayupost.json"), + headers: activitypub_object_headers() }} end @@ -795,7 +829,8 @@ def get( {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/winterdienst_webfinger.json") + body: File.read!("test/fixtures/tesla_mock/winterdienst_webfinger.json"), + headers: activitypub_object_headers() }} end @@ -867,12 +902,21 @@ def get("https://social.heldscal.la/.well-known/host-meta", _, _, _) do end def get("https://mastodon.social/users/lambadalambda", _, _, _) do - {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/lambadalambda.json")}} + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/lambadalambda.json"), + headers: activitypub_object_headers() + }} end def get("https://apfed.club/channel/indio", _, _, _) do {:ok, - %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/osada-user-indio.json")}} + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/osada-user-indio.json"), + headers: activitypub_object_headers() + }} end def get("https://social.heldscal.la/user/23211", _, _, [{"accept", "application/activity+json"}]) do @@ -895,7 +939,8 @@ def get("http://localhost:4001/users/masto_closed/followers", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/users_mock/masto_closed_followers.json") + body: File.read!("test/fixtures/users_mock/masto_closed_followers.json"), + headers: activitypub_object_headers() }} end @@ -903,7 +948,8 @@ def get("http://localhost:4001/users/masto_closed/followers?page=1", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/users_mock/masto_closed_followers_page.json") + body: File.read!("test/fixtures/users_mock/masto_closed_followers_page.json"), + headers: activitypub_object_headers() }} end @@ -911,7 +957,8 @@ def get("http://localhost:4001/users/masto_closed/following", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/users_mock/masto_closed_following.json") + body: File.read!("test/fixtures/users_mock/masto_closed_following.json"), + headers: activitypub_object_headers() }} end @@ -919,7 +966,8 @@ def get("http://localhost:4001/users/masto_closed/following?page=1", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/users_mock/masto_closed_following_page.json") + body: File.read!("test/fixtures/users_mock/masto_closed_following_page.json"), + headers: activitypub_object_headers() }} end @@ -927,7 +975,8 @@ def get("http://localhost:8080/followers/fuser3", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/users_mock/friendica_followers.json") + body: File.read!("test/fixtures/users_mock/friendica_followers.json"), + headers: activitypub_object_headers() }} end @@ -935,7 +984,8 @@ def get("http://localhost:8080/following/fuser3", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/users_mock/friendica_following.json") + body: File.read!("test/fixtures/users_mock/friendica_following.json"), + headers: activitypub_object_headers() }} end @@ -943,7 +993,8 @@ def get("http://localhost:4001/users/fuser2/followers", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/users_mock/pleroma_followers.json") + body: File.read!("test/fixtures/users_mock/pleroma_followers.json"), + headers: activitypub_object_headers() }} end @@ -951,7 +1002,8 @@ def get("http://localhost:4001/users/fuser2/following", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/users_mock/pleroma_following.json") + body: File.read!("test/fixtures/users_mock/pleroma_following.json"), + headers: activitypub_object_headers() }} end @@ -1049,7 +1101,8 @@ def get("https://info.pleroma.site/activity.json", _, _, [ {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/https__info.pleroma.site_activity.json") + body: File.read!("test/fixtures/tesla_mock/https__info.pleroma.site_activity.json"), + headers: activitypub_object_headers() }} end @@ -1063,7 +1116,8 @@ def get("https://info.pleroma.site/activity2.json", _, _, [ {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/https__info.pleroma.site_activity2.json") + body: File.read!("test/fixtures/tesla_mock/https__info.pleroma.site_activity2.json"), + headers: activitypub_object_headers() }} end @@ -1077,7 +1131,8 @@ def get("https://info.pleroma.site/activity3.json", _, _, [ {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/https__info.pleroma.site_activity3.json") + body: File.read!("test/fixtures/tesla_mock/https__info.pleroma.site_activity3.json"), + headers: activitypub_object_headers() }} end @@ -1110,7 +1165,12 @@ def get("https://www.patreon.com/posts/mastodon-2-9-and-28121681", _, _, _) do end def get("http://mastodon.example.org/@admin/99541947525187367", _, _, _) do - {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/mastodon-post-activity.json")}} + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/mastodon-post-activity.json"), + headers: activitypub_object_headers() + }} end def get("https://info.pleroma.site/activity4.json", _, _, _) do @@ -1137,7 +1197,8 @@ def get("https://skippers-bin.com/notes/7x9tmrp97i", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/tesla_mock/misskey_poll_no_end_date.json") + body: File.read!("test/fixtures/tesla_mock/misskey_poll_no_end_date.json"), + headers: activitypub_object_headers() }} end @@ -1146,11 +1207,21 @@ def get("https://example.org/emoji/firedfox.png", _, _, _) do end def get("https://skippers-bin.com/users/7v1w1r8ce6", _, _, _) do - {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/sjw.json")}} + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/sjw.json"), + headers: activitypub_object_headers() + }} end def get("https://patch.cx/users/rin", _, _, _) do - {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/rin.json")}} + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/rin.json"), + headers: activitypub_object_headers() + }} end def get( @@ -1160,12 +1231,20 @@ def get( _ ) do {:ok, - %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/funkwhale_audio.json")}} + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/funkwhale_audio.json"), + headers: activitypub_object_headers() + }} end def get("https://channels.tests.funkwhale.audio/federation/actors/compositions", _, _, _) do {:ok, - %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/funkwhale_channel.json")}} + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/funkwhale_channel.json"), + headers: activitypub_object_headers() + }} end def get("http://example.com/rel_me/error", _, _, _) do @@ -1173,7 +1252,12 @@ def get("http://example.com/rel_me/error", _, _, _) do end def get("https://relay.mastodon.host/actor", _, _, _) do - {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/relay/relay.json")}} + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/relay/relay.json"), + headers: activitypub_object_headers() + }} end def get("http://localhost:4001/", _, "", [{"accept", "text/html"}]) do From c09813193a3429bbab3a3310d9c4b9b75efd6154 Mon Sep 17 00:00:00 2001 From: Michael Walker <mike@barrucadu.co.uk> Date: Thu, 12 Nov 2020 22:20:17 +0000 Subject: [PATCH 71/74] Install file-dev in Dockerfile build stage This is required by the majic, added in #2534. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c210cf79c..4e7c01c5d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ COPY . . ENV MIX_ENV=prod -RUN apk add git gcc g++ musl-dev make cmake &&\ +RUN apk add git gcc g++ musl-dev make cmake file-dev &&\ echo "import Mix.Config" > config/prod.secret.exs &&\ mix local.hex --force &&\ mix local.rebar --force &&\ From 10528344c7c56c70dcfc09a72ff5f109f7b952fe Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Fri, 13 Nov 2020 09:07:08 +0300 Subject: [PATCH 72/74] remove PurgeExpiredActivity from Oban db config --- ...pired_activity_worker_from_oban_config.exs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 priv/repo/migrations/20201113060459_remove_purge_expired_activity_worker_from_oban_config.exs diff --git a/priv/repo/migrations/20201113060459_remove_purge_expired_activity_worker_from_oban_config.exs b/priv/repo/migrations/20201113060459_remove_purge_expired_activity_worker_from_oban_config.exs new file mode 100644 index 000000000..fe31f4442 --- /dev/null +++ b/priv/repo/migrations/20201113060459_remove_purge_expired_activity_worker_from_oban_config.exs @@ -0,0 +1,19 @@ +defmodule Pleroma.Repo.Migrations.RemovePurgeExpiredActivityWorkerFromObanConfig do + use Ecto.Migration + + def change do + with %Pleroma.ConfigDB{} = config <- + Pleroma.ConfigDB.get_by_params(%{group: :pleroma, key: Oban}), + crontab when is_list(crontab) <- config.value[:crontab], + index when is_integer(index) <- + Enum.find_index(crontab, fn {_, worker} -> + worker == Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker + end) do + updated_value = Keyword.put(config.value, :crontab, List.delete_at(crontab, index)) + + config + |> Ecto.Changeset.change(value: updated_value) + |> Pleroma.Repo.update() + end + end +end From 70e4b86250da9ef97a836f497510c36bf22fa905 Mon Sep 17 00:00:00 2001 From: Ilja <pleroma@spectraltheorem.be> Date: Fri, 13 Nov 2020 13:35:46 +0000 Subject: [PATCH 73/74] Make notifs view work for reports * These are the first small steps for issue 2034 "Reports should send a notification to admins". * I added a new type of notification "pleroma:report" to the the database manually (a migration will need to be written later) * I added the new type to the notification_controller * I made the view return the notification. It doesn't include the report itself (yet) --- CHANGELOG.md | 1 + docs/API/differences_in_mastoapi_responses.md | 20 +++++++- lib/pleroma/notification.ex | 12 ++++- .../operations/notification_operation.ex | 3 ++ .../mastodon_api/views/notification_view.ex | 11 +++++ ...eroma_report_to_enum_for_notifications.exs | 48 +++++++++++++++++++ test/pleroma/notification_test.exs | 13 +++++ .../notification_controller_test.exs | 28 +++++++++++ .../views/notification_view_test.exs | 22 +++++++++ 9 files changed, 155 insertions(+), 3 deletions(-) create mode 100644 priv/repo/migrations/20200831152600_add_pleroma_report_to_enum_for_notifications.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index f0b90ff40..c963972c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`) - Mix task option for force-unfollowing relays - Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details). +- Reports now generate notifications for admins and mods. - Pleroma API: Importing the mutes users from CSV files. - Experimental websocket-based federation between Pleroma instances. - Support pagination of blocks and mutes diff --git a/docs/API/differences_in_mastoapi_responses.md b/docs/API/differences_in_mastoapi_responses.md index 3075b6b86..ba48a2ca1 100644 --- a/docs/API/differences_in_mastoapi_responses.md +++ b/docs/API/differences_in_mastoapi_responses.md @@ -129,12 +129,30 @@ The `type` value is `pleroma:emoji_reaction`. Has these fields: - `account`: The account of the user who reacted - `status`: The status that was reacted on +### ChatMention Notification (not default) + +This notification has to be requested explicitly. + +The `type` value is `pleroma:chat_mention` + +- `account`: The account who sent the message +- `chat_message`: The chat message + +### Report Notification (not default) + +This notification has to be requested explicitly. + +The `type` value is `pleroma:report` + +- `account`: The account who reported +- `report`: The report + ## GET `/api/v1/notifications` Accepts additional parameters: - `exclude_visibilities`: will exclude the notifications for activities with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`). Usage example: `GET /api/v1/notifications?exclude_visibilities[]=direct&exclude_visibilities[]=private`. -- `include_types`: will include the notifications for activities with the given types. The parameter accepts an array of types (`mention`, `follow`, `reblog`, `favourite`, `move`, `pleroma:emoji_reaction`). Usage example: `GET /api/v1/notifications?include_types[]=mention&include_types[]=reblog`. +- `include_types`: will include the notifications for activities with the given types. The parameter accepts an array of types (`mention`, `follow`, `reblog`, `favourite`, `move`, `pleroma:emoji_reaction`, `pleroma:chat_mention`, `pleroma:report`). Usage example: `GET /api/v1/notifications?include_types[]=mention&include_types[]=reblog`. ## DELETE `/api/v1/notifications/destroy_multiple` diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 8868a910e..dd7a1c824 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -70,6 +70,7 @@ def unread_notifications_count(%User{id: user_id}) do move pleroma:chat_mention pleroma:emoji_reaction + pleroma:report reblog } @@ -367,7 +368,7 @@ def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = act end def create_notifications(%Activity{data: %{"type" => type}} = activity, options) - when type in ["Follow", "Like", "Announce", "Move", "EmojiReact"] do + when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag"] do do_create_notifications(activity, options) end @@ -410,6 +411,9 @@ defp type_from_activity(%{data: %{"type" => type}} = activity) do "EmojiReact" -> "pleroma:emoji_reaction" + "Flag" -> + "pleroma:report" + # Compatibility with old reactions "EmojiReaction" -> "pleroma:emoji_reaction" @@ -467,7 +471,7 @@ def create_notification(%Activity{} = activity, %User{} = user, do_send \\ true) def get_notified_from_activity(activity, local_only \\ true) def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only) - when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact"] do + when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact", "Flag"] do potential_receiver_ap_ids = get_potential_receiver_ap_ids(activity) potential_receivers = @@ -503,6 +507,10 @@ def get_potential_receiver_ap_ids(%{data: %{"type" => "Follow", "object" => obje [object_id] end + def get_potential_receiver_ap_ids(%{data: %{"type" => "Flag"}}) do + User.all_superusers() |> Enum.map(fn user -> user.ap_id end) + end + def get_potential_receiver_ap_ids(activity) do [] |> Utils.maybe_notify_to_recipients(activity) diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex index f09be64cb..264a530d2 100644 --- a/lib/pleroma/web/api_spec/operations/notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex @@ -193,6 +193,7 @@ defp notification_type do "mention", "pleroma:emoji_reaction", "pleroma:chat_mention", + "pleroma:report", "move", "follow_request" ], @@ -206,6 +207,8 @@ defp notification_type do - `poll` - A poll you have voted in or created has ended - `move` - Someone moved their account - `pleroma:emoji_reaction` - Someone reacted with emoji to your status + - `pleroma:chat_mention` - Someone mentioned you in a chat message + - `pleroma:report` - Someone was reported """ } end diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index c97e6d32f..5b06a6b51 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -11,6 +11,8 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do alias Pleroma.Object alias Pleroma.User alias Pleroma.UserRelationship + alias Pleroma.Web.AdminAPI.Report + alias Pleroma.Web.AdminAPI.ReportView alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.NotificationView @@ -118,11 +120,20 @@ def render( "pleroma:chat_mention" -> put_chat_message(response, activity, reading_user, status_render_opts) + "pleroma:report" -> + put_report(response, activity) + type when type in ["follow", "follow_request"] -> response end end + defp put_report(response, activity) do + report_render = ReportView.render("show.json", Report.extract_report_info(activity)) + + Map.put(response, :report, report_render) + end + defp put_emoji(response, activity) do Map.put(response, :emoji, activity.data["content"]) end diff --git a/priv/repo/migrations/20200831152600_add_pleroma_report_to_enum_for_notifications.exs b/priv/repo/migrations/20200831152600_add_pleroma_report_to_enum_for_notifications.exs new file mode 100644 index 000000000..01fb90459 --- /dev/null +++ b/priv/repo/migrations/20200831152600_add_pleroma_report_to_enum_for_notifications.exs @@ -0,0 +1,48 @@ +defmodule Pleroma.Repo.Migrations.AddPleromaReportTypeToEnumForNotifications do + use Ecto.Migration + + @disable_ddl_transaction true + + def up do + """ + alter type notification_type add value 'pleroma:report' + """ + |> execute() + end + + def down do + alter table(:notifications) do + modify(:type, :string) + end + + """ + delete from notifications where type = 'pleroma:report' + """ + |> execute() + + """ + drop type if exists notification_type + """ + |> execute() + + """ + create type notification_type as enum ( + 'follow', + 'follow_request', + 'mention', + 'move', + 'pleroma:emoji_reaction', + 'pleroma:chat_mention', + 'reblog', + 'favourite' + ) + """ + |> execute() + + """ + alter table notifications + alter column type type notification_type using (type::notification_type) + """ + |> execute() + end +end diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs index 92c0bc8b6..ed2cd219d 100644 --- a/test/pleroma/notification_test.exs +++ b/test/pleroma/notification_test.exs @@ -32,6 +32,19 @@ test "never returns nil" do refute {:ok, [nil]} == Notification.create_notifications(activity) end + test "creates a notification for a report" do + reporting_user = insert(:user) + reported_user = insert(:user) + {:ok, moderator_user} = insert(:user) |> User.admin_api_update(%{is_moderator: true}) + + {:ok, activity} = CommonAPI.report(reporting_user, %{account_id: reported_user.id}) + + {:ok, [notification]} = Notification.create_notifications(activity) + + assert notification.user_id == moderator_user.id + assert notification.type == "pleroma:report" + end + test "creates a notification for an emoji reaction" do user = insert(:user) other_user = insert(:user) diff --git a/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs index 5fd518c60..9ac8488f6 100644 --- a/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs @@ -75,6 +75,34 @@ test "by default, does not contain pleroma:chat_mention" do assert [_] = result end + test "by default, does not contain pleroma:report" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + other_user = insert(:user) + third_user = insert(:user) + + user + |> User.admin_api_update(%{is_moderator: true}) + + {:ok, activity} = CommonAPI.post(other_user, %{status: "hey"}) + + {:ok, _report} = + CommonAPI.report(third_user, %{account_id: other_user.id, status_ids: [activity.id]}) + + result = + conn + |> get("/api/v1/notifications") + |> json_response_and_validate_schema(200) + + assert [] == result + + result = + conn + |> get("/api/v1/notifications?include_types[]=pleroma:report") + |> json_response_and_validate_schema(200) + + assert [_] = result + end + test "getting a single notification" do %{user: user, conn: conn} = oauth_access(["read:notifications"]) other_user = insert(:user) diff --git a/test/pleroma/web/mastodon_api/views/notification_view_test.exs b/test/pleroma/web/mastodon_api/views/notification_view_test.exs index 2f6a808f1..9de11a87e 100644 --- a/test/pleroma/web/mastodon_api/views/notification_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/notification_view_test.exs @@ -12,6 +12,8 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User + alias Pleroma.Web.AdminAPI.Report + alias Pleroma.Web.AdminAPI.ReportView alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.AccountView @@ -207,6 +209,26 @@ test "EmojiReact notification" do test_notifications_rendering([notification], user, [expected]) end + test "Report notification" do + reporting_user = insert(:user) + reported_user = insert(:user) + {:ok, moderator_user} = insert(:user) |> User.admin_api_update(%{is_moderator: true}) + + {:ok, activity} = CommonAPI.report(reporting_user, %{account_id: reported_user.id}) + {:ok, [notification]} = Notification.create_notifications(activity) + + expected = %{ + id: to_string(notification.id), + pleroma: %{is_seen: false, is_muted: false}, + type: "pleroma:report", + account: AccountView.render("show.json", %{user: reporting_user, for: moderator_user}), + created_at: Utils.to_masto_date(notification.inserted_at), + report: ReportView.render("show.json", Report.extract_report_info(activity)) + } + + test_notifications_rendering([notification], moderator_user, [expected]) + end + test "muted notification" do user = insert(:user) another_user = insert(:user) From e2f573d68baa8b161b92459ffacf534054082422 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me> Date: Sat, 14 Nov 2020 22:27:13 +0100 Subject: [PATCH 74/74] pleroma.instance: Fix Exiftool module name --- CHANGELOG.md | 3 +++ lib/mix/tasks/pleroma/instance.ex | 2 +- test/mix/tasks/pleroma/instance_test.exs | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b619bd891..ab73de0ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,9 @@ switched to a new configuration mechanism, however it was not officially removed ### Changed - API: Empty parameter values for integer parameters are now ignored in non-strict validaton mode. +### Fixes +- Config generation: rename `Pleroma.Upload.Filter.ExifTool` to `Pleroma.Upload.Filter.Exiftool` + ## [2.1.2] - 2020-09-17 ### Security diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index fc21ae062..ac8688424 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -284,7 +284,7 @@ defp write_robots_txt(static_dir, indexable, template_dir) do defp upload_filters(filters) when is_map(filters) do enabled_filters = if filters.strip do - [Pleroma.Upload.Filter.ExifTool] + [Pleroma.Upload.Filter.Exiftool] else [] end diff --git a/test/mix/tasks/pleroma/instance_test.exs b/test/mix/tasks/pleroma/instance_test.exs index 8a02710ee..6580fc932 100644 --- a/test/mix/tasks/pleroma/instance_test.exs +++ b/test/mix/tasks/pleroma/instance_test.exs @@ -88,7 +88,7 @@ test "running gen" do assert generated_config =~ "password: \"dbpass\"" assert generated_config =~ "configurable_from_database: true" assert generated_config =~ "http: [ip: {127, 0, 0, 1}, port: 4000]" - assert generated_config =~ "filters: [Pleroma.Upload.Filter.ExifTool]" + assert generated_config =~ "filters: [Pleroma.Upload.Filter.Exiftool]" assert File.read!(tmp_path() <> "setup.psql") == generated_setup_psql() assert File.exists?(Path.expand("./test/instance/static/robots.txt")) end