Allow deleting uploaded files after media migration

Until now only the current base_url was checked.
After a media domain migration all pre-existing files
thus turned undeletable. Fix this with a new config
option allowing to list all old media base urls.
(This may also come in handy for a later db refactor, see
 AkkomaGang/akkoma#765)

CHANGELOG.md

@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - status and user HTML pages now provide ActivityPub alternate links
 - the `prune_objects` mix task no longer deletes pinned posts by default
 - added `--prune-pinned` and `--keep-followed {posts,full,none}` options to the `prune_objects` mix task
+- new config option `Pleroma.Upload, :all_base_urls`
 
 ### Fixed
 - Internal actors no longer pretend to have unresolvable follow(er|ing) collections
@@ -21,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
   this lead e.g. to unlisted replies from Pleroma instances being partially treated as private posts
 - fixed our fetch actor advertising bogus follower and following collection ActivityPub IDs
 - fix network-path references not being handled by media proxy
+- if `Pleroma.Upload, :all_base_urls` is set accordingly, uploaded files can now be deleted after a domain migration
 
 ### Changed
 - Internal and relay actors are now again represented with type "Application"

config/config.exs

@@ -65,6 +65,7 @@
   link_name: false,
   filename_display_max_length: 30,
   base_url: nil,
+  all_base_urls: nil,
   allowed_mime_types: ["image", "audio", "video"]
 
 config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads"

docs/docs/configuration/cheatsheet.md

@@ -606,6 +606,7 @@ the source code is here: [kocaptcha](https://github.com/koto-bank/kocaptcha). Th
 * `link_name`: When enabled Akkoma will add a `name` parameter to the url of the upload, for example `https://instance.tld/media/corndog.png?name=corndog.png`. This is needed to provide the correct filename in Content-Disposition headers
 * `base_url`: The base URL to access a user-uploaded file; MUST be configured explicitly.
   Using a (sub)domain distinct from the instance endpoint is **strongly** recommended. A good value might be `https://media.myakkoma.instance/media/`. 
+* `all_base_url`: list of all base urls ever used *(**both** current and past)*; if unset defaults to a single-entry list containig the current `base_url`
 * `proxy_opts`: Proxy options, see `Pleroma.ReverseProxy` documentation.
 * `filename_display_max_length`: Set max length of a filename to display. 0 = no limit. Default: 30.
 

lib/pleroma/workers/attachments_cleanup_worker.ex

@@ -22,6 +22,7 @@ def enqueue_if_needed(%{
       }) do
     with true <- Config.get([:instance, :cleanup_attachments]),
          true <- URI.parse(actor).host == Pleroma.Web.Endpoint.host(),
+         attachments <- Enum.filter(attachments, &deletable_attachment/1),
          [_ | _] <- attachments do
       enqueue(
         "cleanup_attachments",
@@ -35,6 +36,18 @@ def enqueue_if_needed(%{
 
   def enqueue_if_needed(_), do: {:ok, :skip}
 
+  defp base_urls() do
+    Config.get([Pleroma.Upload, :all_base_urls]) ||
+      [Config.get!([Pleroma.Upload, :base_url])]
+  end
+
+  defp deletable_attachment(%{"id" => _id, "url" => [%{"href" => href} | _]}) do
+    # We can't delete files later if we can't strip the prefix
+    Enum.any?(base_urls(), fn url -> String.starts_with?(href, url) end)
+  end
+
+  defp deletable_attachment(_), do: false
+
   @impl Oban.Worker
   def perform(%Job{
         args: %{
@@ -66,18 +79,28 @@ def perform(%Job{
 
   def perform(%Job{args: %{"op" => "cleanup_attachments", "object" => _object}}), do: {:ok, :skip}
 
+  defp trim_first_leading(string, []), do: string
+
+  defp trim_first_leading(string, [prefix | rest]) do
+    trimmed = String.trim_leading(string, prefix)
+
+    if trimmed != string do
+      trimmed
+    else
+      trim_first_leading(string, rest)
+    end
+  end
+
   defp do_clean({object_ids, attachment_urls}) do
     uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
 
-    base_url =
-      String.trim_trailing(
-        Pleroma.Upload.base_url(),
-        "/"
-      )
+    base_urls =
+      base_urls()
+      |> Enum.map(fn url -> String.trim_trailing(url, "/") end)
 
     Enum.each(attachment_urls, fn href ->
       href
-      |> String.trim_leading("#{base_url}")
+      |> trim_first_leading(base_urls)
       |> uploader.delete_file()
     end)
 

test/pleroma/workers/attachments_cleanup_worker_test.exs

@@ -14,6 +14,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorkerTest do
 
   setup do
     clear_config([:instance, :cleanup_attachments], true)
+    clear_config([Pleroma.Upload, :all_base_urls])
 
     file = %Plug.Upload{
       content_type: "image/jpeg",
@@ -83,4 +84,24 @@ test "doesn't delete immediately", %{attachment: attachment, user: user} do
     assert Object.get_by_id(attachment.id) == nil
     refute File.exists?(path)
   end
+
+  test "skips localpost with unmappable URLs", %{attachment: attachment, user: user} do
+    local_url = Pleroma.Web.Endpoint.url()
+
+    attach_data =
+      attachment.data
+      |> Map.update!("url", fn
+        [%{"href" => _} = url | _] ->
+          [%{url | "href" => "https://oldmedia.example/files/123.png"}]
+      end)
+
+    local_data = %{
+      "id" => local_url <> "/obj/123",
+      "actor" => user.ap_id,
+      "content" => "content",
+      "attachment" => [attach_data]
+    }
+
+    assert {:ok, :skip} = AttachmentsCleanupWorker.enqueue_if_needed(local_data)
+  end
 end