added media proxy invalidation

This commit is contained in:
Maksim Pechnikov 2020-05-15 21:34:46 +03:00
parent aeacfb2479
commit cb40602a16
6 changed files with 101 additions and 30 deletions

View file

@ -378,6 +378,13 @@ config :pleroma, :rich_media,
config :pleroma, :media_proxy,
enabled: false,
invalidation: [
enabled: false,
provider: Pleroma.Web.MediaProxy.Invalidation.Script,
options: %{
script_path: ""
}
],
proxy_opts: [
redirect_on_failure: false,
max_body_length: 25 * 1_048_576,

View file

@ -9,11 +9,13 @@ defmodule Pleroma.Object do
import Ecto.Changeset
alias Pleroma.Activity
alias Pleroma.Config
alias Pleroma.Object
alias Pleroma.Object.Fetcher
alias Pleroma.ObjectTombstone
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Workers.AttachmentsCleanupWorker
require Logger
@ -183,27 +185,37 @@ defmodule Pleroma.Object do
def delete(%Object{data: %{"id" => id}} = object) do
with {:ok, _obj} = swap_object_with_tombstone(object),
deleted_activity = Activity.delete_all_by_object_ap_id(id),
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
{:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
with true <- Pleroma.Config.get([:instance, :cleanup_attachments]) do
{:ok, _} =
Pleroma.Workers.AttachmentsCleanupWorker.enqueue("cleanup_attachments", %{
"object" => object
})
end
{:ok, _} <- invalid_object_cache(object) do
cleanup_attachments(
Config.get([:instance, :cleanup_attachments]),
%{"object" => object}
)
{:ok, object, deleted_activity}
end
end
def prune(%Object{data: %{"id" => id}} = object) do
@spec cleanup_attachments(boolean(), %{required(:object) => map()}) ::
{:ok, Oban.Job.t() | nil}
def cleanup_attachments(true, %{"object" => _} = params) do
AttachmentsCleanupWorker.enqueue("cleanup_attachments", params)
end
def cleanup_attachments(_, _), do: {:ok, nil}
def prune(%Object{data: %{"id" => _id}} = object) do
with {:ok, object} <- Repo.delete(object),
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
{:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
{:ok, _} <- invalid_object_cache(object) do
{:ok, object}
end
end
def invalid_object_cache(%Object{data: %{"id" => id}}) do
with {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
Cachex.del(:web_resp_cache, URI.parse(id).path)
end
end
def set_cache(%Object{data: %{"id" => ap_id}} = object) do
Cachex.put(:object_cache, "object:#{ap_id}", object)
{:ok, object}

View file

@ -0,0 +1,19 @@
defmodule Pleroma.Web.MediaProxy.Invalidation do
@callback purge(list(String.t()), map()) :: {:ok, String.t()} | {:error, String.t()}
alias Pleroma.Config
def purge(urls) do
[:media_proxy, :invalidation, :enabled]
|> Config.get()
|> do_purge(urls)
end
defp do_purge(true, urls) do
config = Config.get([:media_proxy, :invalidation])
config[:provider].purge(urls, config[:options])
:ok
end
defp do_purge(_, _), do: :ok
end

View file

@ -0,0 +1,12 @@
defmodule Pleroma.Web.MediaProxy.Invalidation.Nginx do
@behaviour Pleroma.Web.MediaProxy.Invalidation
@impl Pleroma.Web.MediaProxy.Invalidation
def purge(urls, _opts) do
Enum.each(urls, fn url ->
Pleroma.HTTP.request(:purge, url, "", [], [])
end)
{:ok, "success"}
end
end

View file

@ -0,0 +1,10 @@
defmodule Pleroma.Web.MediaProxy.Invalidation.Script do
@behaviour Pleroma.Web.MediaProxy.Invalidation
@impl Pleroma.Web.MediaProxy.Invalidation
def purge(urls, %{script_path: script_path} = options) do
script_args = List.wrap(Map.get(options, :script_args, []))
System.cmd(Path.expand(script_path), [urls] ++ script_args)
{:ok, "success"}
end
end

View file

@ -27,8 +27,20 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
prefix =
case Pleroma.Config.get([Pleroma.Upload, :base_url]) do
nil -> "media"
_ -> ""
end
base_url =
String.trim_trailing(
Pleroma.Config.get([Pleroma.Upload, :base_url], Pleroma.Web.base_url()),
"/"
)
# find all objects for copies of the attachments, name and actor doesn't matter here
delete_ids =
object_ids_and_hrefs =
from(o in Object,
where:
fragment(
@ -67,29 +79,28 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
|> Enum.map(fn {href, %{id: id, count: count}} ->
# only delete files that have single instance
with 1 <- count do
prefix =
case Pleroma.Config.get([Pleroma.Upload, :base_url]) do
nil -> "media"
_ -> ""
end
href
|> String.trim_leading("#{base_url}/#{prefix}")
|> uploader.delete_file()
base_url =
String.trim_trailing(
Pleroma.Config.get([Pleroma.Upload, :base_url], Pleroma.Web.base_url()),
"/"
)
file_path = String.trim_leading(href, "#{base_url}/#{prefix}")
uploader.delete_file(file_path)
{id, href}
else
_ -> {id, nil}
end
id
end)
from(o in Object, where: o.id in ^delete_ids)
object_ids = Enum.map(object_ids_and_hrefs, fn {id, _} -> id end)
from(o in Object, where: o.id in ^object_ids)
|> Repo.delete_all()
object_ids_and_hrefs
|> Enum.filter(fn {_, href} -> not is_nil(href) end)
|> Enum.map(&elem(&1, 1))
|> Pleroma.Web.MediaProxy.Invalidation.purge()
{:ok, :success}
end
def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: :ok
def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip}
end