Allow deleting uploaded files after media migration
Until now only the current base_url was checked. After a media domain migration all pre-existing files thus turned undeletable. Fix this with a new config option allowing to list all old media base urls. (This may also come in handy for a later db refactor, see #765)
This commit is contained in:
parent
6283155e21
commit
c3e5ac30ed
4 changed files with 52 additions and 6 deletions
|
@ -66,6 +66,7 @@ config :pleroma, Pleroma.Upload,
|
||||||
proxy_remote: false,
|
proxy_remote: false,
|
||||||
filename_display_max_length: 30,
|
filename_display_max_length: 30,
|
||||||
base_url: nil,
|
base_url: nil,
|
||||||
|
all_base_urls: nil,
|
||||||
allowed_mime_types: ["image", "audio", "video"]
|
allowed_mime_types: ["image", "audio", "video"]
|
||||||
|
|
||||||
config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads"
|
config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads"
|
||||||
|
|
|
@ -605,6 +605,7 @@ the source code is here: [kocaptcha](https://github.com/koto-bank/kocaptcha). Th
|
||||||
* `link_name`: When enabled Akkoma will add a `name` parameter to the url of the upload, for example `https://instance.tld/media/corndog.png?name=corndog.png`. This is needed to provide the correct filename in Content-Disposition headers
|
* `link_name`: When enabled Akkoma will add a `name` parameter to the url of the upload, for example `https://instance.tld/media/corndog.png?name=corndog.png`. This is needed to provide the correct filename in Content-Disposition headers
|
||||||
* `base_url`: The base URL to access a user-uploaded file; MUST be configured explicitly.
|
* `base_url`: The base URL to access a user-uploaded file; MUST be configured explicitly.
|
||||||
Using a (sub)domain distinct from the instance endpoint is **strongly** recommended. A good value might be `https://media.myakkoma.instance/media/`.
|
Using a (sub)domain distinct from the instance endpoint is **strongly** recommended. A good value might be `https://media.myakkoma.instance/media/`.
|
||||||
|
* `all_base_url`: list of all base urls ever used *(**both** current and past)*; if unset defaults to a single-entry list containig the current `base_url`
|
||||||
* `proxy_remote`: If you're using a remote uploader, Akkoma will proxy media requests instead of redirecting to it.
|
* `proxy_remote`: If you're using a remote uploader, Akkoma will proxy media requests instead of redirecting to it.
|
||||||
* `proxy_opts`: Proxy options, see `Pleroma.ReverseProxy` documentation.
|
* `proxy_opts`: Proxy options, see `Pleroma.ReverseProxy` documentation.
|
||||||
* `filename_display_max_length`: Set max length of a filename to display. 0 = no limit. Default: 30.
|
* `filename_display_max_length`: Set max length of a filename to display. 0 = no limit. Default: 30.
|
||||||
|
|
|
@ -22,6 +22,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
|
||||||
}) do
|
}) do
|
||||||
with true <- Config.get([:instance, :cleanup_attachments]),
|
with true <- Config.get([:instance, :cleanup_attachments]),
|
||||||
true <- URI.parse(actor).host == Pleroma.Web.Endpoint.host(),
|
true <- URI.parse(actor).host == Pleroma.Web.Endpoint.host(),
|
||||||
|
attachments <- Enum.filter(attachments, &deletable_attachment/1),
|
||||||
[_ | _] <- attachments do
|
[_ | _] <- attachments do
|
||||||
enqueue("cleanup_attachments", %{"actor" => actor, "attachments" => attachments})
|
enqueue("cleanup_attachments", %{"actor" => actor, "attachments" => attachments})
|
||||||
else
|
else
|
||||||
|
@ -31,6 +32,18 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
|
||||||
|
|
||||||
def enqueue_if_needed(_), do: {:ok, :skip}
|
def enqueue_if_needed(_), do: {:ok, :skip}
|
||||||
|
|
||||||
|
defp base_urls() do
|
||||||
|
Config.get([Pleroma.Upload, :all_base_urls]) ||
|
||||||
|
[Config.get!([Pleroma.Upload, :base_url])]
|
||||||
|
end
|
||||||
|
|
||||||
|
defp deletable_attachment(%{"id" => _id, "url" => [%{"href" => href} | _]}) do
|
||||||
|
# We can't delete files later if we can't strip the prefix
|
||||||
|
Enum.any?(base_urls(), fn url -> String.starts_with?(href, url) end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp deletable_attachment(_), do: false
|
||||||
|
|
||||||
@impl Oban.Worker
|
@impl Oban.Worker
|
||||||
def perform(%Job{
|
def perform(%Job{
|
||||||
args: %{
|
args: %{
|
||||||
|
@ -62,18 +75,28 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
|
||||||
|
|
||||||
def perform(%Job{args: %{"op" => "cleanup_attachments", "object" => _object}}), do: {:ok, :skip}
|
def perform(%Job{args: %{"op" => "cleanup_attachments", "object" => _object}}), do: {:ok, :skip}
|
||||||
|
|
||||||
|
defp trim_first_leading(string, []), do: string
|
||||||
|
|
||||||
|
defp trim_first_leading(string, [prefix | rest]) do
|
||||||
|
trimmed = String.trim_leading(string, prefix)
|
||||||
|
|
||||||
|
if trimmed != string do
|
||||||
|
trimmed
|
||||||
|
else
|
||||||
|
trim_first_leading(string, rest)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
defp do_clean({object_ids, attachment_urls}) do
|
defp do_clean({object_ids, attachment_urls}) do
|
||||||
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
|
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
|
||||||
|
|
||||||
base_url =
|
base_urls =
|
||||||
String.trim_trailing(
|
base_urls()
|
||||||
Pleroma.Upload.base_url(),
|
|> Enum.map(fn url -> String.trim_trailing(url, "/") end)
|
||||||
"/"
|
|
||||||
)
|
|
||||||
|
|
||||||
Enum.each(attachment_urls, fn href ->
|
Enum.each(attachment_urls, fn href ->
|
||||||
href
|
href
|
||||||
|> String.trim_leading("#{base_url}")
|
|> trim_first_leading(base_urls)
|
||||||
|> uploader.delete_file()
|
|> uploader.delete_file()
|
||||||
end)
|
end)
|
||||||
|
|
||||||
|
|
|
@ -12,6 +12,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorkerTest do
|
||||||
|
|
||||||
setup do
|
setup do
|
||||||
clear_config([:instance, :cleanup_attachments], true)
|
clear_config([:instance, :cleanup_attachments], true)
|
||||||
|
clear_config([Pleroma.Upload, :all_base_urls])
|
||||||
|
|
||||||
file = %Plug.Upload{
|
file = %Plug.Upload{
|
||||||
content_type: "image/jpeg",
|
content_type: "image/jpeg",
|
||||||
|
@ -57,4 +58,24 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorkerTest do
|
||||||
|
|
||||||
assert {:ok, %Oban.Job{}} = AttachmentsCleanupWorker.enqueue_if_needed(local_data)
|
assert {:ok, %Oban.Job{}} = AttachmentsCleanupWorker.enqueue_if_needed(local_data)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
test "skips localpost with unmappable URLs", %{attachment: attachment, user: user} do
|
||||||
|
local_url = Pleroma.Web.Endpoint.url()
|
||||||
|
|
||||||
|
attach_data =
|
||||||
|
attachment.data
|
||||||
|
|> Map.update!("url", fn
|
||||||
|
[%{"href" => _} = url | _] ->
|
||||||
|
[%{url | "href" => "https://oldmedia.example/files/123.png"}]
|
||||||
|
end)
|
||||||
|
|
||||||
|
local_data = %{
|
||||||
|
"id" => local_url <> "/obj/123",
|
||||||
|
"actor" => user.ap_id,
|
||||||
|
"content" => "content",
|
||||||
|
"attachment" => [attach_data]
|
||||||
|
}
|
||||||
|
|
||||||
|
assert {:ok, :skip} = AttachmentsCleanupWorker.enqueue_if_needed(local_data)
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
Loading…
Reference in a new issue