From 2e7c5fe2ded095c95f8596970d8fc3aaf0128f1b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 8 Aug 2020 12:33:37 -0500 Subject: [PATCH 01/11] Add migration to remove invalid activity expirations --- .../20200808173046_only_expire_creates.exs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 priv/repo/migrations/20200808173046_only_expire_creates.exs diff --git a/priv/repo/migrations/20200808173046_only_expire_creates.exs b/priv/repo/migrations/20200808173046_only_expire_creates.exs new file mode 100644 index 000000000..5a34dc7c1 --- /dev/null +++ b/priv/repo/migrations/20200808173046_only_expire_creates.exs @@ -0,0 +1,20 @@ +defmodule Pleroma.Repo.Migrations.OnlyExpireCreates do + use Ecto.Migration + + def up do + statement = """ + DELETE FROM + activity_expirations A USING activities B + WHERE + A.activity_id = B.id + AND B.local = false + AND B.data->>'type' != 'Create'; + """ + + execute(statement) + end + + def down do + :ok + end +end From cf4c97242be588e55dfccb37ab2c3d6dcac3cb0f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 8 Aug 2020 12:40:52 -0500 Subject: [PATCH 02/11] Ensure we only expire Create activities with the Mix task --- lib/mix/tasks/pleroma/database.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index d57e59b11..b2dc3d0f3 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -136,6 +136,7 @@ defmodule Mix.Tasks.Pleroma.Database do |> join(:left, [a], u in assoc(a, :expiration)) |> where(local: true) |> where([a, u], is_nil(u)) + |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data)) |> Pleroma.RepoStreamer.chunk_stream(100) |> Stream.each(fn activities -> Enum.each(activities, fn activity -> From 761cc5b4a2b4c0ef610ae7296f614ec4c9ceccad Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 8 Aug 2020 12:44:18 -0500 Subject: [PATCH 03/11] Don't filter on local --- priv/repo/migrations/20200808173046_only_expire_creates.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/priv/repo/migrations/20200808173046_only_expire_creates.exs b/priv/repo/migrations/20200808173046_only_expire_creates.exs index 5a34dc7c1..42fb73375 100644 --- a/priv/repo/migrations/20200808173046_only_expire_creates.exs +++ b/priv/repo/migrations/20200808173046_only_expire_creates.exs @@ -7,7 +7,6 @@ defmodule Pleroma.Repo.Migrations.OnlyExpireCreates do activity_expirations A USING activities B WHERE A.activity_id = B.id - AND B.local = false AND B.data->>'type' != 'Create'; """ From e08ea01d09c67a93801aa05d33bad0eb24dfca8b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 8 Aug 2020 12:49:02 -0500 Subject: [PATCH 04/11] Limit expirations for each cron execution to 50. This should prevent servers from being crushed. 50/min is a pretty good rate. --- lib/pleroma/activity_expiration.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/activity_expiration.ex b/lib/pleroma/activity_expiration.ex index 7cc9668b3..84edf68ef 100644 --- a/lib/pleroma/activity_expiration.ex +++ b/lib/pleroma/activity_expiration.ex @@ -46,6 +46,7 @@ defmodule Pleroma.ActivityExpiration do ActivityExpiration |> where([exp], exp.scheduled_at < ^naive_datetime) + |> limit(50) |> Repo.all() end From 7e4932362b7e672b08543cfd189deb3776268fe3 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 11 Aug 2020 10:54:38 +0200 Subject: [PATCH 05/11] SideEffects: Handle strange deletion case. --- lib/pleroma/web/activity_pub/side_effects.ex | 12 ++++++++++-- test/web/activity_pub/side_effects_test.exs | 19 ++++++++++++++++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 5104d38ee..bff7c6629 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -22,6 +22,8 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do alias Pleroma.Web.Streamer alias Pleroma.Workers.BackgroundWorker + require Logger + def handle(object, meta \\ []) # Tasks this handle @@ -217,13 +219,15 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do # - Stream out the activity def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object, meta) do deleted_object = - Object.normalize(deleted_object, false) || User.get_cached_by_ap_id(deleted_object) + Object.normalize(deleted_object, false) || + User.get_cached_by_ap_id(deleted_object) result = case deleted_object do %Object{} -> with {:ok, deleted_object, activity} <- Object.delete(deleted_object), - %User{} = user <- User.get_cached_by_ap_id(deleted_object.data["actor"]) do + {_, actor} when is_binary(actor) <- {:actor, deleted_object.data["actor"]}, + %User{} = user <- User.get_cached_by_ap_id(actor) do User.remove_pinnned_activity(user, activity) {:ok, user} = ActivityPub.decrease_note_count_if_public(user, deleted_object) @@ -237,6 +241,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do ActivityPub.stream_out(object) ActivityPub.stream_out_participations(deleted_object, user) :ok + else + {:actor, _} -> + Logger.error("The object doesn't have an actor: #{inspect(deleted_object)}") + :no_object_actor end %User{} -> diff --git a/test/web/activity_pub/side_effects_test.exs b/test/web/activity_pub/side_effects_test.exs index 4a08eb7ee..9efbaad04 100644 --- a/test/web/activity_pub/side_effects_test.exs +++ b/test/web/activity_pub/side_effects_test.exs @@ -19,8 +19,9 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do alias Pleroma.Web.ActivityPub.SideEffects alias Pleroma.Web.CommonAPI - import Pleroma.Factory + import ExUnit.CaptureLog import Mock + import Pleroma.Factory describe "handle_after_transaction" do test "it streams out notifications and streams" do @@ -221,6 +222,22 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do assert User.get_cached_by_ap_id(user.ap_id).deactivated end + + test "it logs issues with objects deletion", %{ + delete: delete, + object: object + } do + {:ok, object} = + object + |> Object.change(%{data: Map.delete(object.data, "actor")}) + |> Repo.update() + + Object.invalid_object_cache(object) + + assert capture_log(fn -> + {:error, :no_object_actor} = SideEffects.handle(delete) + end) =~ "object doesn't have an actor" + end end describe "EmojiReact objects" do From 724ed354f25fb83f65dff2fbadd4b5a121fb77d0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 11 Aug 2020 11:28:22 -0500 Subject: [PATCH 06/11] Ensure only Note objects are set to expire --- lib/mix/tasks/pleroma/database.ex | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index b2dc3d0f3..7d8f00b08 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -134,14 +134,23 @@ defmodule Mix.Tasks.Pleroma.Database do Pleroma.Activity |> join(:left, [a], u in assoc(a, :expiration)) + |> join(:inner, [a, _u], o in Object, + on: + fragment( + "(?->>'id') = COALESCE((?)->'object'->> 'id', (?)->>'object')", + o.data, + a.data, + a.data + ) + ) |> where(local: true) |> where([a, u], is_nil(u)) |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data)) + |> where([_a, _u, o], fragment("?->>'type' = 'Note'", o.data)) |> Pleroma.RepoStreamer.chunk_stream(100) |> Stream.each(fn activities -> Enum.each(activities, fn activity -> expires_at = Timex.shift(activity.inserted_at, days: days) - Pleroma.ActivityExpiration.create(activity, expires_at, false) end) end) From 57b455de5a378d661eb794c2c9d75a2684d74ef3 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 12 Aug 2020 12:41:47 +0300 Subject: [PATCH 07/11] leave expirations with Create and Note types --- priv/repo/migrations/20200808173046_only_expire_creates.exs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/priv/repo/migrations/20200808173046_only_expire_creates.exs b/priv/repo/migrations/20200808173046_only_expire_creates.exs index 42fb73375..9df52956f 100644 --- a/priv/repo/migrations/20200808173046_only_expire_creates.exs +++ b/priv/repo/migrations/20200808173046_only_expire_creates.exs @@ -4,10 +4,10 @@ defmodule Pleroma.Repo.Migrations.OnlyExpireCreates do def up do statement = """ DELETE FROM - activity_expirations A USING activities B + activity_expirations a_exp USING activities a, objects o WHERE - A.activity_id = B.id - AND B.data->>'type' != 'Create'; + a_exp.activity_id = a.id AND (o.data->>'id') = COALESCE(a.data->'object'->>'id', a.data->>'object') + AND (a.data->>'type' != 'Create' OR o.data->>'type' != 'Note'); """ execute(statement) From eec1ba232c42285fc69c26b5ccc32c504955eab5 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 12 Aug 2020 15:15:17 +0300 Subject: [PATCH 08/11] don't expire pinned posts --- lib/mix/tasks/pleroma/database.ex | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 7d8f00b08..0142071a8 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -150,8 +150,12 @@ defmodule Mix.Tasks.Pleroma.Database do |> Pleroma.RepoStreamer.chunk_stream(100) |> Stream.each(fn activities -> Enum.each(activities, fn activity -> - expires_at = Timex.shift(activity.inserted_at, days: days) - Pleroma.ActivityExpiration.create(activity, expires_at, false) + user = User.get_cached_by_ap_id(activity.actor) + + if activity.id not in user.pinned_activities do + expires_at = Timex.shift(activity.inserted_at, days: days) + Pleroma.ActivityExpiration.create(activity, expires_at, false) + end end) end) |> Stream.run() From 091da10832b35ce55e5d7a5a3655d4cdb98bb27c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 12 Aug 2020 11:00:01 -0500 Subject: [PATCH 09/11] Add the ActivityExpirationPolicy MRF to docs and clarify post expiration criteria. --- docs/configuration/cheatsheet.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index ca587af8e..e5742bc3a 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -114,6 +114,7 @@ To add configuration to your config file, you can copy it from the base config. * `Pleroma.Web.ActivityPub.MRF.MentionPolicy`: Drops posts mentioning configurable users. (See [`:mrf_mention`](#mrf_mention)). * `Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`: Restricts activities to a configured set of vocabulary. (See [`:mrf_vocabulary`](#mrf_vocabulary)). * `Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy`: Rejects or delists posts based on their age when received. (See [`:mrf_object_age`](#mrf_object_age)). + * `Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy`: Sets a default expiration on all posts made by users of the local instance. Requires `Pleroma.ActivityExpiration` to be enabled for processing the scheduled delections. * `transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo). * `transparency_exclusions`: Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value. @@ -220,6 +221,8 @@ config :pleroma, :mrf_user_allowlist, %{ ## Pleroma.ActivityExpiration +Enables the worker which processes posts scheduled for deletion. Pinned posts are exempt from expiration. + * `enabled`: whether expired activities will be sent to the job queue to be deleted ## Frontends From 3ab83f837eb9c454b91374c6aeb14b37e8fdd3b1 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 12 Aug 2020 19:46:47 +0300 Subject: [PATCH 10/11] don't load pinned activities in due_expirations --- lib/mix/tasks/pleroma/database.ex | 4 +--- lib/pleroma/activity.ex | 6 ++++++ lib/pleroma/activity_expiration.ex | 4 ++++ test/activity_expiration_test.exs | 5 ++++- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 0142071a8..22a325b47 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -150,9 +150,7 @@ defmodule Mix.Tasks.Pleroma.Database do |> Pleroma.RepoStreamer.chunk_stream(100) |> Stream.each(fn activities -> Enum.each(activities, fn activity -> - user = User.get_cached_by_ap_id(activity.actor) - - if activity.id not in user.pinned_activities do + if not Pleroma.Activity.pinned_by_actor?(activity) do expires_at = Timex.shift(activity.inserted_at, days: days) Pleroma.ActivityExpiration.create(activity, expires_at, false) end diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index c3cea8d2a..97feebeaa 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -340,4 +340,10 @@ defmodule Pleroma.Activity do _ -> nil end end + + @spec pinned_by_actor?(Activity.t()) :: boolean() + def pinned_by_actor?(%Activity{} = activity) do + actor = user_actor(activity) + activity.id in actor.pinned_activities + end end diff --git a/lib/pleroma/activity_expiration.ex b/lib/pleroma/activity_expiration.ex index 84edf68ef..955f0578e 100644 --- a/lib/pleroma/activity_expiration.ex +++ b/lib/pleroma/activity_expiration.ex @@ -47,7 +47,11 @@ defmodule Pleroma.ActivityExpiration do ActivityExpiration |> where([exp], exp.scheduled_at < ^naive_datetime) |> limit(50) + |> preload(:activity) |> Repo.all() + |> Enum.reject(fn %{activity: activity} -> + Activity.pinned_by_actor?(activity) + end) end def validate_scheduled_at(changeset, false), do: changeset diff --git a/test/activity_expiration_test.exs b/test/activity_expiration_test.exs index d75c06cc7..f86d79826 100644 --- a/test/activity_expiration_test.exs +++ b/test/activity_expiration_test.exs @@ -11,7 +11,10 @@ defmodule Pleroma.ActivityExpirationTest do test "finds activities due to be deleted only" do activity = insert(:note_activity) - expiration_due = insert(:expiration_in_the_past, %{activity_id: activity.id}) + + expiration_due = + insert(:expiration_in_the_past, %{activity_id: activity.id}) |> Repo.preload(:activity) + activity2 = insert(:note_activity) insert(:expiration_in_the_future, %{activity_id: activity2.id}) From 29a7bcd5bbb3a39fe1b31c9f5ffc0077f23fc101 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 12 Aug 2020 20:01:21 +0300 Subject: [PATCH 11/11] reverting pinned posts in filtering --- lib/mix/tasks/pleroma/database.ex | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 22a325b47..7d8f00b08 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -150,10 +150,8 @@ defmodule Mix.Tasks.Pleroma.Database do |> Pleroma.RepoStreamer.chunk_stream(100) |> Stream.each(fn activities -> Enum.each(activities, fn activity -> - if not Pleroma.Activity.pinned_by_actor?(activity) do - expires_at = Timex.shift(activity.inserted_at, days: days) - Pleroma.ActivityExpiration.create(activity, expires_at, false) - end + expires_at = Timex.shift(activity.inserted_at, days: days) + Pleroma.ActivityExpiration.create(activity, expires_at, false) end) end) |> Stream.run()