Add --keep-non-public option
The prune_objects task already did this by default, but is undocumented. Now we require an explicit parameter for it. The parameter also works in combination with --keep-threads Docs still needs to happen
This commit is contained in:
ilja
parent
04cc1d41ce
commit
92d2f8b401
2 changed files with 177 additions and 23 deletions
|
|
@ -68,7 +68,8 @@ def run(["prune_objects" | args]) do
|
|||
args,
|
||||
strict: [
|
||||
vacuum: :boolean,
|
||||
keep_threads: :boolean
|
||||
keep_threads: :boolean,
|
||||
keep_non_public: :boolean
|
||||
]
|
||||
)
|
||||
|
||||
|
|
@ -77,41 +78,79 @@ def run(["prune_objects" | args]) do
|
|||
deadline = Pleroma.Config.get([:instance, :remote_post_retention_days])
|
||||
time_deadline = NaiveDateTime.utc_now() |> NaiveDateTime.add(-(deadline * 86_400))
|
||||
|
||||
if Keyword.get(options, :keep_threads) do
|
||||
Logger.info(
|
||||
"Pruning objects older than #{deadline} days without local interaction, keeping threads intact"
|
||||
)
|
||||
log_message = "Pruning objects older than #{deadline} days"
|
||||
|
||||
log_message =
|
||||
if Keyword.get(options, :keep_non_public) do
|
||||
log_message <> ", keeping non public posts"
|
||||
else
|
||||
log_message
|
||||
end
|
||||
|
||||
log_message =
|
||||
if Keyword.get(options, :keep_threads) do
|
||||
log_message <> ", keeping threads intact"
|
||||
else
|
||||
log_message
|
||||
end
|
||||
|
||||
Logger.info(log_message)
|
||||
|
||||
if Keyword.get(options, :keep_threads) do
|
||||
# We want to delete objects from threads where
|
||||
# 1. the newest post is still old
|
||||
# 2. none of the activities is local
|
||||
# 3. none of the activities is bookmarked
|
||||
# 4. optionally none of the posts is non-public
|
||||
deletable_context =
|
||||
Pleroma.Activity
|
||||
|> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id)
|
||||
|> group_by([a], fragment("? ->> 'context'::text", a.data))
|
||||
if Keyword.get(options, :keep_non_public) do
|
||||
Pleroma.Activity
|
||||
|> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id)
|
||||
|> group_by([a], fragment("? ->> 'context'::text", a.data))
|
||||
|> having(
|
||||
[a],
|
||||
not fragment(
|
||||
# Posts (checked on Create Activity) is non-public
|
||||
"bool_or((not(?->'to' \\? ? OR ?->'cc' \\? ?)) and ? ->> 'type' = 'Create')",
|
||||
a.data,
|
||||
^Pleroma.Constants.as_public(),
|
||||
a.data,
|
||||
^Pleroma.Constants.as_public(),
|
||||
a.data
|
||||
)
|
||||
)
|
||||
else
|
||||
Pleroma.Activity
|
||||
|> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id)
|
||||
|> group_by([a], fragment("? ->> 'context'::text", a.data))
|
||||
end
|
||||
|> having([a], max(a.updated_at) < ^time_deadline)
|
||||
|> having([a], not fragment("bool_or(?)", a.local))
|
||||
|> having([a, b], fragment("max(?::text) is null", b.id))
|
||||
|> having([_, b], fragment("max(?::text) is null", b.id))
|
||||
|> select([a], fragment("? ->> 'context'::text", a.data))
|
||||
|
||||
Pleroma.Object
|
||||
|> where([o], fragment("? ->> 'context'::text", o.data) in subquery(deletable_context))
|
||||
else
|
||||
Logger.info("Pruning objects older than #{deadline} days")
|
||||
|
||||
from(o in Object,
|
||||
where:
|
||||
if Keyword.get(options, :keep_non_public) do
|
||||
Pleroma.Object
|
||||
|> where(
|
||||
[o],
|
||||
fragment(
|
||||
"?->'to' \\? ? OR ?->'cc' \\? ?",
|
||||
o.data,
|
||||
^Pleroma.Constants.as_public(),
|
||||
o.data,
|
||||
^Pleroma.Constants.as_public()
|
||||
),
|
||||
where: o.inserted_at < ^time_deadline,
|
||||
where:
|
||||
fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host())
|
||||
)
|
||||
)
|
||||
else
|
||||
Pleroma.Object
|
||||
end
|
||||
|> where([o], o.updated_at < ^time_deadline)
|
||||
|> where(
|
||||
[o],
|
||||
fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host())
|
||||
)
|
||||
end
|
||||
|> Repo.delete_all(timeout: :infinity)
|
||||
|
|
|
|||
|
|
@ -46,7 +46,6 @@ test "it replaces objects with references" do
|
|||
|
||||
describe "prune_objects" do
|
||||
test "it prunes old objects from the database" do
|
||||
insert(:note)
|
||||
deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + 1
|
||||
|
||||
date =
|
||||
|
|
@ -55,18 +54,134 @@ test "it prunes old objects from the database" do
|
|||
|> Timex.to_naive_datetime()
|
||||
|> NaiveDateTime.truncate(:second)
|
||||
|
||||
%{id: id} =
|
||||
insert(:note)
|
||||
|
||||
%{id: note_remote_public_id} =
|
||||
:note
|
||||
|> insert()
|
||||
|> Ecto.Changeset.change(%{inserted_at: date})
|
||||
|> Ecto.Changeset.change(%{updated_at: date})
|
||||
|> Repo.update!()
|
||||
|
||||
assert length(Repo.all(Object)) == 2
|
||||
note_remote_non_public =
|
||||
%{id: note_remote_non_public_id, data: note_remote_non_public_data} =
|
||||
:note
|
||||
|> insert()
|
||||
|
||||
note_remote_non_public
|
||||
|> Ecto.Changeset.change(%{
|
||||
updated_at: date,
|
||||
data: note_remote_non_public_data |> update_in(["to"], fn _ -> [] end)
|
||||
})
|
||||
|> Repo.update!()
|
||||
|
||||
assert length(Repo.all(Object)) == 3
|
||||
|
||||
Mix.Tasks.Pleroma.Database.run(["prune_objects"])
|
||||
|
||||
assert length(Repo.all(Object)) == 1
|
||||
refute Object.get_by_id(id)
|
||||
refute Object.get_by_id(note_remote_public_id)
|
||||
refute Object.get_by_id(note_remote_non_public_id)
|
||||
end
|
||||
|
||||
test "with the --keep-non-public option it still keeps non-public posts even if they are not local" do
|
||||
deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + 1
|
||||
|
||||
date =
|
||||
Timex.now()
|
||||
|> Timex.shift(days: -deadline)
|
||||
|> Timex.to_naive_datetime()
|
||||
|> NaiveDateTime.truncate(:second)
|
||||
|
||||
insert(:note)
|
||||
|
||||
%{id: note_remote_id} =
|
||||
:note
|
||||
|> insert()
|
||||
|> Ecto.Changeset.change(%{updated_at: date})
|
||||
|> Repo.update!()
|
||||
|
||||
note_remote_non_public =
|
||||
%{data: note_remote_non_public_data} =
|
||||
:note
|
||||
|> insert()
|
||||
|
||||
note_remote_non_public
|
||||
|> Ecto.Changeset.change(%{
|
||||
updated_at: date,
|
||||
data: note_remote_non_public_data |> update_in(["to"], fn _ -> [] end)
|
||||
})
|
||||
|> Repo.update!()
|
||||
|
||||
assert length(Repo.all(Object)) == 3
|
||||
|
||||
Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-non-public"])
|
||||
|
||||
assert length(Repo.all(Object)) == 2
|
||||
refute Object.get_by_id(note_remote_id)
|
||||
end
|
||||
|
||||
test "with the --keep-threads and --keep-non-public option it keeps old threads with non-public replies even if the interaction is not local" do
|
||||
# For non-public we only check Create Activities because only these are relevant for threads
|
||||
# Flags are always non-public, Announces from relays can be non-public...
|
||||
deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + 1
|
||||
|
||||
old_insert_date =
|
||||
Timex.now()
|
||||
|> Timex.shift(days: -deadline)
|
||||
|> Timex.to_naive_datetime()
|
||||
|> NaiveDateTime.truncate(:second)
|
||||
|
||||
remote_user1 = insert(:user, local: false)
|
||||
remote_user2 = insert(:user, local: false)
|
||||
|
||||
# Old remote non-public reply (should be kept)
|
||||
{:ok, old_remote_post1_activity} =
|
||||
CommonAPI.post(remote_user1, %{status: "some thing", local: false})
|
||||
|
||||
old_remote_post1_activity
|
||||
|> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
|
||||
|> Repo.update!()
|
||||
|
||||
{:ok, old_remote_non_public_reply_activity} =
|
||||
CommonAPI.post(remote_user2, %{
|
||||
status: "some reply",
|
||||
in_reply_to_status_id: old_remote_post1_activity.id
|
||||
})
|
||||
|
||||
old_remote_non_public_reply_activity
|
||||
|> Ecto.Changeset.change(%{
|
||||
local: false,
|
||||
updated_at: old_insert_date,
|
||||
data: old_remote_non_public_reply_activity.data |> update_in(["to"], fn _ -> [] end)
|
||||
})
|
||||
|> Repo.update!()
|
||||
|
||||
# Old remote non-public Announce (should be removed)
|
||||
{:ok, old_remote_post2_activity = %{data: %{"object" => old_remote_post2_id}}} =
|
||||
CommonAPI.post(remote_user1, %{status: "some thing", local: false})
|
||||
|
||||
old_remote_post2_activity
|
||||
|> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
|
||||
|> Repo.update!()
|
||||
|
||||
{:ok, old_remote_non_public_repeat_activity} =
|
||||
CommonAPI.repeat(old_remote_post2_activity.id, remote_user2)
|
||||
|
||||
old_remote_non_public_repeat_activity
|
||||
|> Ecto.Changeset.change(%{
|
||||
local: false,
|
||||
updated_at: old_insert_date,
|
||||
data: old_remote_non_public_repeat_activity.data |> update_in(["to"], fn _ -> [] end)
|
||||
})
|
||||
|> Repo.update!()
|
||||
|
||||
assert length(Repo.all(Object)) == 3
|
||||
|
||||
Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads", "--keep-non-public"])
|
||||
|
||||
Repo.all(Pleroma.Activity)
|
||||
assert length(Repo.all(Object)) == 2
|
||||
refute Object.get_by_ap_id(old_remote_post2_id)
|
||||
end
|
||||
|
||||
test "with the --keep-threads option it still keeps non-old threads even with no local interactions" do
|
||||
|
|
@ -143,7 +258,7 @@ test "with the --keep-threads option it deletes old threads with no local intera
|
|||
|
||||
Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads"])
|
||||
|
||||
assert Repo.all(Object) == []
|
||||
assert length(Repo.all(Object)) == 0
|
||||
end
|
||||
|
||||
test "with the --keep-threads option it keeps old threads with local interaction" do
|
||||
|
|
|
|||
Loading…
Reference in a new issue