added media proxy invalidation

This commit is contained in:
Maksim Pechnikov 2020-05-15 21:34:46 +03:00
parent aeacfb2479
commit cb40602a16
6 changed files with 101 additions and 30 deletions

View file

@ -378,6 +378,13 @@ config :pleroma, :rich_media,
config :pleroma, :media_proxy, config :pleroma, :media_proxy,
enabled: false, enabled: false,
invalidation: [
enabled: false,
provider: Pleroma.Web.MediaProxy.Invalidation.Script,
options: %{
script_path: ""
}
],
proxy_opts: [ proxy_opts: [
redirect_on_failure: false, redirect_on_failure: false,
max_body_length: 25 * 1_048_576, max_body_length: 25 * 1_048_576,

View file

@ -9,11 +9,13 @@ defmodule Pleroma.Object do
import Ecto.Changeset import Ecto.Changeset
alias Pleroma.Activity alias Pleroma.Activity
alias Pleroma.Config
alias Pleroma.Object alias Pleroma.Object
alias Pleroma.Object.Fetcher alias Pleroma.Object.Fetcher
alias Pleroma.ObjectTombstone alias Pleroma.ObjectTombstone
alias Pleroma.Repo alias Pleroma.Repo
alias Pleroma.User alias Pleroma.User
alias Pleroma.Workers.AttachmentsCleanupWorker
require Logger require Logger
@ -183,27 +185,37 @@ defmodule Pleroma.Object do
def delete(%Object{data: %{"id" => id}} = object) do def delete(%Object{data: %{"id" => id}} = object) do
with {:ok, _obj} = swap_object_with_tombstone(object), with {:ok, _obj} = swap_object_with_tombstone(object),
deleted_activity = Activity.delete_all_by_object_ap_id(id), deleted_activity = Activity.delete_all_by_object_ap_id(id),
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), {:ok, _} <- invalid_object_cache(object) do
{:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do cleanup_attachments(
with true <- Pleroma.Config.get([:instance, :cleanup_attachments]) do Config.get([:instance, :cleanup_attachments]),
{:ok, _} = %{"object" => object}
Pleroma.Workers.AttachmentsCleanupWorker.enqueue("cleanup_attachments", %{ )
"object" => object
})
end
{:ok, object, deleted_activity} {:ok, object, deleted_activity}
end end
end end
def prune(%Object{data: %{"id" => id}} = object) do @spec cleanup_attachments(boolean(), %{required(:object) => map()}) ::
{:ok, Oban.Job.t() | nil}
def cleanup_attachments(true, %{"object" => _} = params) do
AttachmentsCleanupWorker.enqueue("cleanup_attachments", params)
end
def cleanup_attachments(_, _), do: {:ok, nil}
def prune(%Object{data: %{"id" => _id}} = object) do
with {:ok, object} <- Repo.delete(object), with {:ok, object} <- Repo.delete(object),
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), {:ok, _} <- invalid_object_cache(object) do
{:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
{:ok, object} {:ok, object}
end end
end end
def invalid_object_cache(%Object{data: %{"id" => id}}) do
with {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
Cachex.del(:web_resp_cache, URI.parse(id).path)
end
end
def set_cache(%Object{data: %{"id" => ap_id}} = object) do def set_cache(%Object{data: %{"id" => ap_id}} = object) do
Cachex.put(:object_cache, "object:#{ap_id}", object) Cachex.put(:object_cache, "object:#{ap_id}", object)
{:ok, object} {:ok, object}

View file

@ -0,0 +1,19 @@
defmodule Pleroma.Web.MediaProxy.Invalidation do
@callback purge(list(String.t()), map()) :: {:ok, String.t()} | {:error, String.t()}
alias Pleroma.Config
def purge(urls) do
[:media_proxy, :invalidation, :enabled]
|> Config.get()
|> do_purge(urls)
end
defp do_purge(true, urls) do
config = Config.get([:media_proxy, :invalidation])
config[:provider].purge(urls, config[:options])
:ok
end
defp do_purge(_, _), do: :ok
end

View file

@ -0,0 +1,12 @@
defmodule Pleroma.Web.MediaProxy.Invalidation.Nginx do
@behaviour Pleroma.Web.MediaProxy.Invalidation
@impl Pleroma.Web.MediaProxy.Invalidation
def purge(urls, _opts) do
Enum.each(urls, fn url ->
Pleroma.HTTP.request(:purge, url, "", [], [])
end)
{:ok, "success"}
end
end

View file

@ -0,0 +1,10 @@
defmodule Pleroma.Web.MediaProxy.Invalidation.Script do
@behaviour Pleroma.Web.MediaProxy.Invalidation
@impl Pleroma.Web.MediaProxy.Invalidation
def purge(urls, %{script_path: script_path} = options) do
script_args = List.wrap(Map.get(options, :script_args, []))
System.cmd(Path.expand(script_path), [urls] ++ script_args)
{:ok, "success"}
end
end

View file

@ -27,8 +27,20 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
prefix =
case Pleroma.Config.get([Pleroma.Upload, :base_url]) do
nil -> "media"
_ -> ""
end
base_url =
String.trim_trailing(
Pleroma.Config.get([Pleroma.Upload, :base_url], Pleroma.Web.base_url()),
"/"
)
# find all objects for copies of the attachments, name and actor doesn't matter here # find all objects for copies of the attachments, name and actor doesn't matter here
delete_ids = object_ids_and_hrefs =
from(o in Object, from(o in Object,
where: where:
fragment( fragment(
@ -67,29 +79,28 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
|> Enum.map(fn {href, %{id: id, count: count}} -> |> Enum.map(fn {href, %{id: id, count: count}} ->
# only delete files that have single instance # only delete files that have single instance
with 1 <- count do with 1 <- count do
prefix = href
case Pleroma.Config.get([Pleroma.Upload, :base_url]) do |> String.trim_leading("#{base_url}/#{prefix}")
nil -> "media" |> uploader.delete_file()
_ -> ""
end
base_url = {id, href}
String.trim_trailing( else
Pleroma.Config.get([Pleroma.Upload, :base_url], Pleroma.Web.base_url()), _ -> {id, nil}
"/"
)
file_path = String.trim_leading(href, "#{base_url}/#{prefix}")
uploader.delete_file(file_path)
end end
id
end) end)
from(o in Object, where: o.id in ^delete_ids) object_ids = Enum.map(object_ids_and_hrefs, fn {id, _} -> id end)
from(o in Object, where: o.id in ^object_ids)
|> Repo.delete_all() |> Repo.delete_all()
object_ids_and_hrefs
|> Enum.filter(fn {_, href} -> not is_nil(href) end)
|> Enum.map(&elem(&1, 1))
|> Pleroma.Web.MediaProxy.Invalidation.purge()
{:ok, :success}
end end
def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: :ok def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip}
end end