Allow deleting uploaded files after media migration

Until now only the current base_url was checked.
After a media domain migration all pre-existing files
thus turned undeletable. Fix this with a new config
option allowing to list all old media base urls.
(This may also come in handy for a later db refactor, see
 #765)

config/config.exs

@@ -66,6 +66,7 @@ config :pleroma, Pleroma.Upload,
   proxy_remote: false,
   filename_display_max_length: 30,
   base_url: nil,
+  all_base_urls: nil,
   allowed_mime_types: ["image", "audio", "video"]
 
 config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads"

docs/docs/configuration/cheatsheet.md

@@ -605,6 +605,7 @@ the source code is here: [kocaptcha](https://github.com/koto-bank/kocaptcha). Th
 * `link_name`: When enabled Akkoma will add a `name` parameter to the url of the upload, for example `https://instance.tld/media/corndog.png?name=corndog.png`. This is needed to provide the correct filename in Content-Disposition headers
 * `base_url`: The base URL to access a user-uploaded file; MUST be configured explicitly.
   Using a (sub)domain distinct from the instance endpoint is **strongly** recommended. A good value might be `https://media.myakkoma.instance/media/`. 
+* `all_base_url`: list of all base urls ever used *(**both** current and past)*; if unset defaults to a single-entry list containig the current `base_url`
 * `proxy_remote`: If you're using a remote uploader, Akkoma will proxy media requests instead of redirecting to it.
 * `proxy_opts`: Proxy options, see `Pleroma.ReverseProxy` documentation.
 * `filename_display_max_length`: Set max length of a filename to display. 0 = no limit. Default: 30.

lib/pleroma/workers/attachments_cleanup_worker.ex

@@ -22,6 +22,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
       }) do
     with true <- Config.get([:instance, :cleanup_attachments]),
          true <- URI.parse(actor).host == Pleroma.Web.Endpoint.host(),
+         attachments <- Enum.filter(attachments, &deletable_attachment/1),
          [_ | _] <- attachments do
       enqueue("cleanup_attachments", %{"actor" => actor, "attachments" => attachments})
     else
@@ -31,6 +32,18 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
 
   def enqueue_if_needed(_), do: {:ok, :skip}
 
+  defp base_urls() do
+    Config.get([Pleroma.Upload, :all_base_urls]) ||
+      [Config.get!([Pleroma.Upload, :base_url])]
+  end
+
+  defp deletable_attachment(%{"id" => _id, "url" => [%{"href" => href} | _]}) do
+    # We can't delete files later if we can't strip the prefix
+    Enum.any?(base_urls(), fn url -> String.starts_with?(href, url) end)
+  end
+
+  defp deletable_attachment(_), do: false
+
   @impl Oban.Worker
   def perform(%Job{
         args: %{
@@ -62,18 +75,28 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
 
   def perform(%Job{args: %{"op" => "cleanup_attachments", "object" => _object}}), do: {:ok, :skip}
 
+  defp trim_first_leading(string, []), do: string
+
+  defp trim_first_leading(string, [prefix | rest]) do
+    trimmed = String.trim_leading(string, prefix)
+
+    if trimmed != string do
+      trimmed
+    else
+      trim_first_leading(string, rest)
+    end
+  end
+
   defp do_clean({object_ids, attachment_urls}) do
     uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
 
-    base_url =
-      String.trim_trailing(
-        Pleroma.Upload.base_url(),
-        "/"
-      )
+    base_urls =
+      base_urls()
+      |> Enum.map(fn url -> String.trim_trailing(url, "/") end)
 
     Enum.each(attachment_urls, fn href ->
       href
-      |> String.trim_leading("#{base_url}")
+      |> trim_first_leading(base_urls)
       |> uploader.delete_file()
     end)
 

test/pleroma/workers/attachments_cleanup_worker_test.exs

@@ -12,6 +12,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorkerTest do
 
   setup do
     clear_config([:instance, :cleanup_attachments], true)
+    clear_config([Pleroma.Upload, :all_base_urls])
 
     file = %Plug.Upload{
       content_type: "image/jpeg",
@@ -57,4 +58,24 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorkerTest do
 
     assert {:ok, %Oban.Job{}} = AttachmentsCleanupWorker.enqueue_if_needed(local_data)
   end
+
+  test "skips localpost with unmappable URLs", %{attachment: attachment, user: user} do
+    local_url = Pleroma.Web.Endpoint.url()
+
+    attach_data =
+      attachment.data
+      |> Map.update!("url", fn
+        [%{"href" => _} = url | _] ->
+          [%{url | "href" => "https://oldmedia.example/files/123.png"}]
+      end)
+
+    local_data = %{
+      "id" => local_url <> "/obj/123",
+      "actor" => user.ap_id,
+      "content" => "content",
+      "attachment" => [attach_data]
+    }
+
+    assert {:ok, :skip} = AttachmentsCleanupWorker.enqueue_if_needed(local_data)
+  end
 end