Merge branch 'fix/backup-url-on-s3' into 'develop'

Standardize base url for uploads

See merge request pleroma/pleroma!3246
This commit is contained in:
feld 2021-01-12 20:54:30 +00:00
commit 5453038ce9
7 changed files with 54 additions and 42 deletions

View file

@ -31,6 +31,7 @@ defmodule Pleroma.Upload do
""" """
alias Ecto.UUID alias Ecto.UUID
alias Pleroma.Config
require Logger require Logger
@type source :: @type source ::
@ -228,4 +229,39 @@ defmodule Pleroma.Upload do
end end
defp url_from_spec(_upload, _base_url, {:url, url}), do: url defp url_from_spec(_upload, _base_url, {:url, url}), do: url
def base_url do
uploader = Config.get([Pleroma.Upload, :uploader])
upload_base_url = Config.get([Pleroma.Upload, :base_url])
public_endpoint = Config.get([uploader, :public_endpoint])
case uploader do
Pleroma.Uploaders.Local ->
upload_base_url || Pleroma.Web.base_url() <> "/media/"
Pleroma.Uploaders.S3 ->
bucket = Config.get([Pleroma.Uploaders.S3, :bucket])
bucket_with_namespace =
cond do
truncated_namespace = Config.get([Pleroma.Uploaders.S3, :truncated_namespace]) ->
truncated_namespace
namespace = Config.get([Pleroma.Uploaders.S3, :bucket_namespace]) ->
namespace <> ":" <> bucket
true ->
bucket
end
if public_endpoint do
Path.join([public_endpoint, bucket_with_namespace])
else
Path.join([upload_base_url, bucket_with_namespace])
end
_ ->
public_endpoint || upload_base_url
end
end
end end

View file

@ -12,26 +12,10 @@ defmodule Pleroma.Uploaders.S3 do
# links with less strict filenames # links with less strict filenames
@impl true @impl true
def get_file(file) do def get_file(file) do
config = Config.get([__MODULE__])
bucket = Keyword.fetch!(config, :bucket)
bucket_with_namespace =
cond do
truncated_namespace = Keyword.get(config, :truncated_namespace) ->
truncated_namespace
namespace = Keyword.get(config, :bucket_namespace) ->
namespace <> ":" <> bucket
true ->
bucket
end
{:ok, {:ok,
{:url, {:url,
Path.join([ Path.join([
Keyword.fetch!(config, :public_endpoint), Pleroma.Upload.base_url(),
bucket_with_namespace,
strict_encode(URI.decode(file)) strict_encode(URI.decode(file))
])}} ])}}
end end

View file

@ -69,7 +69,7 @@ defmodule Pleroma.Web.MediaProxy do
# non-local non-whitelisted URLs through it and be sure that body size constraint is preserved. # non-local non-whitelisted URLs through it and be sure that body size constraint is preserved.
def preview_enabled?, do: enabled?() and !!Config.get([:media_preview_proxy, :enabled]) def preview_enabled?, do: enabled?() and !!Config.get([:media_preview_proxy, :enabled])
def local?(url), do: String.starts_with?(url, Pleroma.Web.base_url()) def local?(url), do: String.starts_with?(url, Web.base_url())
def whitelisted?(url) do def whitelisted?(url) do
%{host: domain} = URI.parse(url) %{host: domain} = URI.parse(url)
@ -77,17 +77,10 @@ defmodule Pleroma.Web.MediaProxy do
mediaproxy_whitelist_domains = mediaproxy_whitelist_domains =
[:media_proxy, :whitelist] [:media_proxy, :whitelist]
|> Config.get() |> Config.get()
|> Kernel.++(["#{Upload.base_url()}"])
|> Enum.map(&maybe_get_domain_from_url/1) |> Enum.map(&maybe_get_domain_from_url/1)
whitelist_domains = domain in mediaproxy_whitelist_domains
if base_url = Config.get([Upload, :base_url]) do
%{host: base_domain} = URI.parse(base_url)
[base_domain | mediaproxy_whitelist_domains]
else
mediaproxy_whitelist_domains
end
domain in whitelist_domains
end end
defp maybe_get_domain_from_url("http" <> _ = url) do defp maybe_get_domain_from_url("http" <> _ = url) do

View file

@ -24,6 +24,6 @@ defmodule Pleroma.Web.PleromaAPI.BackupView do
end end
def download_url(%Backup{file_name: file_name}) do def download_url(%Backup{file_name: file_name}) do
Pleroma.Web.Endpoint.url() <> "/media/backups/" <> file_name Pleroma.Upload.base_url() <> "/backups/" <> file_name
end end
end end

View file

@ -62,7 +62,7 @@ defmodule Pleroma.Web.Plugs.UploadedMedia do
def call(conn, _opts), do: conn def call(conn, _opts), do: conn
defp media_is_banned(%{request_path: path} = _conn, {:static_dir, _}) do defp media_is_banned(%{request_path: path} = _conn, {:static_dir, _}) do
MediaProxy.in_banned_urls(Pleroma.Web.base_url() <> path) MediaProxy.in_banned_urls(Pleroma.Upload.base_url() <> path)
end end
defp media_is_banned(_, {:url, url}), do: MediaProxy.in_banned_urls(url) defp media_is_banned(_, {:url, url}), do: MediaProxy.in_banned_urls(url)

View file

@ -32,21 +32,15 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
defp do_clean({object_ids, attachment_urls}) do defp do_clean({object_ids, attachment_urls}) do
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
prefix =
case Pleroma.Config.get([Pleroma.Upload, :base_url]) do
nil -> "media"
_ -> ""
end
base_url = base_url =
String.trim_trailing( String.trim_trailing(
Pleroma.Config.get([Pleroma.Upload, :base_url], Pleroma.Web.base_url()), Pleroma.Upload.base_url(),
"/" "/"
) )
Enum.each(attachment_urls, fn href -> Enum.each(attachment_urls, fn href ->
href href
|> String.trim_leading("#{base_url}/#{prefix}") |> String.trim_leading("#{base_url}")
|> uploader.delete_file() |> uploader.delete_file()
end) end)

View file

@ -11,11 +11,16 @@ defmodule Pleroma.Uploaders.S3Test do
import Mock import Mock
import ExUnit.CaptureLog import ExUnit.CaptureLog
setup do: setup do
clear_config(Pleroma.Uploaders.S3, clear_config(Pleroma.Upload,
bucket: "test_bucket", uploader: Pleroma.Uploaders.S3
public_endpoint: "https://s3.amazonaws.com" )
)
clear_config(Pleroma.Uploaders.S3,
bucket: "test_bucket",
public_endpoint: "https://s3.amazonaws.com"
)
end
describe "get_file/1" do describe "get_file/1" do
test "it returns path to local folder for files" do test "it returns path to local folder for files" do