Add period pruning of old data

config/config.exs

@@ -569,7 +569,8 @@ config :pleroma, Oban,
     new_users_digest: 1,
     mute_expire: 5,
     search_indexing: 10,
-    nodeinfo_fetcher: 1
+    nodeinfo_fetcher: 1,
+    database_prune: 1
   ],
   plugins: [
     Oban.Plugins.Pruner,
@@ -577,7 +578,8 @@ config :pleroma, Oban,
   ],
   crontab: [
     {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker},
-    {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker}
+    {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker},
+    {"0 3 * * *", Pleroma.Workers.Cron.PruneDatabaseWorker}
   ]
 
 config :pleroma, :workers,
@@ -605,7 +607,8 @@ config :pleroma, :workers,
     new_users_digest: :timer.seconds(10),
     mute_expire: :timer.seconds(5),
     search_indexing: :timer.seconds(5),
-    nodeinfo_fetcher: :timer.seconds(10)
+    nodeinfo_fetcher: :timer.seconds(10),
+    database_prune: :timer.minutes(10)
   ]
 
 config :pleroma, Pleroma.Formatter,

lib/mix/tasks/pleroma/database.ex

@@ -110,6 +110,14 @@ defmodule Mix.Tasks.Pleroma.Database do
     end
   end
 
+  def run(["prune_task"]) do
+    start_pleroma()
+
+    nil
+    |> Pleroma.Workers.Cron.PruneDatabaseWorker.perform()
+    |> IO.inspect()
+  end
+
   def run(["fix_likes_collections"]) do
     start_pleroma()
 

lib/pleroma/activity/pruner.ex

@@ -0,0 +1,23 @@
+defmodule Pleroma.Activity.Pruner do
+  @moduledoc """
+  Prunes activities from the database.
+  """
+  @cutoff 30
+
+  alias Pleroma.Activity
+  alias Pleroma.Repo
+  import Ecto.Query
+
+  def prune_deletes do
+    before_time = cutoff()
+
+    from(a in Activity,
+      where: fragment("?->>'type' = ?", a.data, "Delete") and a.inserted_at < ^before_time
+    )
+    |> Repo.delete_all(timeout: :infinity)
+  end
+
+  defp cutoff do
+    DateTime.utc_now() |> Timex.shift(days: -@cutoff)
+  end
+end

lib/pleroma/object/pruner.ex

@@ -0,0 +1,33 @@
+defmodule Pleroma.Object.Pruner do
+  @moduledoc """
+  Prunes objects from the database.
+  """
+  @cutoff 30
+
+  alias Pleroma.Object
+  alias Pleroma.Delivery
+  alias Pleroma.Repo
+  import Ecto.Query
+
+  def prune_tombstoned_deliveries do
+    before_time = cutoff()
+
+    from(d in Delivery)
+    |> join(:inner, [d], o in Object, on: d.object_id == o.id)
+    |> where([d, o], fragment("?->>'type' = ?", o.data, "Tombstone"))
+    |> Repo.delete_all(timeout: :infinity)
+  end
+
+  def prune_tombstones do
+    before_time = cutoff()
+
+    from(o in Object,
+      where: fragment("?->>'type' = ?", o.data, "Tombstone") and o.inserted_at < ^before_time
+    )
+    |> Repo.delete_all(timeout: :infinity, on_delete: :delete_all)
+  end
+
+  defp cutoff do
+    DateTime.utc_now() |> Timex.shift(days: -@cutoff)
+  end
+end

lib/pleroma/workers/cron/database_prune_worker.ex

@@ -0,0 +1,26 @@
+defmodule Pleroma.Workers.Cron.PruneDatabaseWorker do
+  @moduledoc """
+  The worker to prune old data from the database.
+  """
+  require Logger
+  use Oban.Worker, queue: "database_prune"
+
+  alias Pleroma.Activity.Pruner, as: ActivityPruner
+  alias Pleroma.Object.Pruner, as: ObjectPruner
+
+  @impl Oban.Worker
+  def perform(_job) do
+    Logger.info("Pruning old data from the database")
+
+    Logger.info("Pruning old deletes")
+    ActivityPruner.prune_deletes()
+
+    Logger.info("Pruning old tombstone delivery entries")
+    ObjectPruner.prune_tombstoned_deliveries()
+
+    Logger.info("Pruning old tombstones")
+    ObjectPruner.prune_tombstones()
+
+    :ok
+  end
+end

test/pleroma/activity/pruner_test.exs

@@ -0,0 +1,29 @@
+defmodule Pleroma.Activity.PrunerTest do
+  use Pleroma.DataCase, async: true
+
+  alias Pleroma.Activity
+  alias Pleroma.Object
+  alias Pleroma.Repo
+  alias Pleroma.Activity.Pruner
+
+  import Pleroma.Factory
+
+  describe "prune_deletes" do
+    test "it prunes old delete objects" do
+      user = insert(:user)
+
+      new_delete = insert(:delete_activity, type: "Delete", user: user)
+
+      old_delete =
+        insert(:delete_activity,
+          type: "Delete",
+          user: user,
+          inserted_at: DateTime.utc_now() |> DateTime.add(-31 * 24, :hour)
+        )
+
+      Pruner.prune_deletes()
+      assert Activity.get_by_id(new_delete.id)
+      refute Activity.get_by_id(old_delete.id)
+    end
+  end
+end

test/pleroma/object/pruner_test.exs

@@ -0,0 +1,24 @@
+defmodule Pleroma.Object.PrunerTest do
+  use Pleroma.DataCase, async: true
+
+  alias Pleroma.Object
+  alias Pleroma.Repo
+  alias Pleroma.Object.Pruner
+
+  import Pleroma.Factory
+
+  describe "prune_deletes" do
+    test "it prunes old delete objects" do
+      new_tombstone = insert(:tombstone)
+
+      old_tombstone =
+        insert(:tombstone,
+          inserted_at: DateTime.utc_now() |> DateTime.add(-31 * 24, :hour)
+        )
+
+      Pruner.prune_tombstones()
+      assert Object.get_by_id(new_tombstone.id)
+      refute Object.get_by_id(old_tombstone.id)
+    end
+  end
+end

test/support/factory.ex

@@ -233,7 +233,7 @@ defmodule Pleroma.Factory do
     %Pleroma.Object{data: Map.merge(data, %{"type" => "Article"})}
   end
 
-  def tombstone_factory do
+  def tombstone_factory(attrs) do
     data = %{
       "type" => "Tombstone",
       "id" => Pleroma.Web.ActivityPub.Utils.generate_object_id(),
@@ -244,6 +244,7 @@ defmodule Pleroma.Factory do
     %Pleroma.Object{
       data: data
     }
+    |> merge_attributes(attrs)
   end
 
   def question_factory(attrs \\ %{}) do
@@ -493,22 +494,22 @@ defmodule Pleroma.Factory do
     }
   end
 
-  def question_activity_factory(attrs \\ %{}) do
+  def delete_activity_factory(attrs \\ %{}) do
     user = attrs[:user] || insert(:user)
-    question = attrs[:question] || insert(:question, user: user)
+    note_activity = attrs[:note_activity] || insert(:note_activity, user: user)
 
     data_attrs = attrs[:data_attrs] || %{}
-    attrs = Map.drop(attrs, [:user, :question, :data_attrs])
+    attrs = Map.drop(attrs, [:user, :data_attrs])
 
     data =
       %{
         "id" => Pleroma.Web.ActivityPub.Utils.generate_activity_id(),
-        "type" => "Create",
-        "actor" => question.data["actor"],
-        "to" => question.data["to"],
-        "object" => question.data["id"],
+        "type" => "Delete",
+        "actor" => note_activity.data["actor"],
+        "to" => note_activity.data["to"],
+        "object" => note_activity.data["id"],
         "published" => DateTime.utc_now() |> DateTime.to_iso8601(),
-        "context" => question.data["context"]
+        "context" => note_activity.data["context"]
       }
       |> Map.merge(data_attrs)