Merge branch 'feature/1922-media-proxy-whitelist' into 'develop'

Support for hosts with scheme in MediaProxy whitelist setting

Closes #1922

See merge request pleroma/pleroma!2754
This commit is contained in:
feld 2020-07-14 18:07:44 +00:00
commit 3f65f2ea79
11 changed files with 282 additions and 199 deletions

View file

@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- MFR policy to set global expiration for all local Create activities - MFR policy to set global expiration for all local Create activities
- OGP rich media parser merged with TwitterCard - OGP rich media parser merged with TwitterCard
- Configuration: `:instance, rewrite_policy` moved to `:mrf, policies`, `:instance, :mrf_transparency` moved to `:mrf, :transparency`, `:instance, :mrf_transparency_exclusions` moved to `:mrf, :transparency_exclusions`. Old config namespace is deprecated. - Configuration: `:instance, rewrite_policy` moved to `:mrf, policies`, `:instance, :mrf_transparency` moved to `:mrf, :transparency`, `:instance, :mrf_transparency_exclusions` moved to `:mrf, :transparency_exclusions`. Old config namespace is deprecated.
- Configuration: `:media_proxy, whitelist` format changed to host with scheme (e.g. `http://example.com` instead of `example.com`). Domain format is deprecated.
<details> <details>
<summary>API Changes</summary> <summary>API Changes</summary>

View file

@ -1768,8 +1768,8 @@ config :pleroma, :config_description, [
%{ %{
key: :whitelist, key: :whitelist,
type: {:list, :string}, type: {:list, :string},
description: "List of domains to bypass the mediaproxy", description: "List of hosts with scheme to bypass the mediaproxy",
suggestions: ["example.com"] suggestions: ["http://example.com"]
} }
] ]
}, },

View file

@ -113,6 +113,11 @@ config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: true
config :pleroma, :instances_favicons, enabled: true config :pleroma, :instances_favicons, enabled: true
config :pleroma, Pleroma.Uploaders.S3,
bucket: nil,
streaming_enabled: true,
public_endpoint: nil
if File.exists?("./config/test.secret.exs") do if File.exists?("./config/test.secret.exs") do
import_config "test.secret.exs" import_config "test.secret.exs"
else else

View file

@ -252,6 +252,7 @@ This section describe PWA manifest instance-specific values. Currently this opti
* `background_color`: Describe the background color of the app. (Example: `"#191b22"`, `"aliceblue"`). * `background_color`: Describe the background color of the app. (Example: `"#191b22"`, `"aliceblue"`).
## :emoji ## :emoji
* `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]` * `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]`
* `pack_extensions`: A list of file extensions for emojis, when no emoji.txt for a pack is present. Example `[".png", ".gif"]` * `pack_extensions`: A list of file extensions for emojis, when no emoji.txt for a pack is present. Example `[".png", ".gif"]`
* `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]` * `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]`
@ -260,13 +261,14 @@ This section describe PWA manifest instance-specific values. Currently this opti
memory for this amount of seconds multiplied by the number of files. memory for this amount of seconds multiplied by the number of files.
## :media_proxy ## :media_proxy
* `enabled`: Enables proxying of remote media to the instances proxy * `enabled`: Enables proxying of remote media to the instances proxy
* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts. * `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts.
* `proxy_opts`: All options defined in `Pleroma.ReverseProxy` documentation, defaults to `[max_body_length: (25*1_048_576)]`. * `proxy_opts`: All options defined in `Pleroma.ReverseProxy` documentation, defaults to `[max_body_length: (25*1_048_576)]`.
* `whitelist`: List of domains to bypass the mediaproxy * `whitelist`: List of hosts with scheme to bypass the mediaproxy (e.g. `https://example.com`)
* `invalidation`: options for remove media from cache after delete object: * `invalidation`: options for remove media from cache after delete object:
* `enabled`: Enables purge cache * `enabled`: Enables purge cache
* `provider`: Which one of the [purge cache strategy](#purge-cache-strategy) to use. * `provider`: Which one of the [purge cache strategy](#purge-cache-strategy) to use.
### Purge cache strategy ### Purge cache strategy
@ -278,6 +280,7 @@ Urls of attachments pass to script as arguments.
* `script_path`: path to external script. * `script_path`: path to external script.
Example: Example:
```elixir ```elixir
config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script,
script_path: "./installation/nginx-cache-purge.example" script_path: "./installation/nginx-cache-purge.example"

View file

@ -54,6 +54,7 @@ defmodule Pleroma.Config.DeprecationWarnings do
check_hellthread_threshold() check_hellthread_threshold()
mrf_user_allowlist() mrf_user_allowlist()
check_old_mrf_config() check_old_mrf_config()
check_media_proxy_whitelist_config()
end end
def check_old_mrf_config do def check_old_mrf_config do
@ -65,7 +66,7 @@ defmodule Pleroma.Config.DeprecationWarnings do
move_namespace_and_warn(@mrf_config_map, warning_preface) move_namespace_and_warn(@mrf_config_map, warning_preface)
end end
@spec move_namespace_and_warn([config_map()], String.t()) :: :ok @spec move_namespace_and_warn([config_map()], String.t()) :: :ok | nil
def move_namespace_and_warn(config_map, warning_preface) do def move_namespace_and_warn(config_map, warning_preface) do
warning = warning =
Enum.reduce(config_map, "", fn Enum.reduce(config_map, "", fn
@ -84,4 +85,16 @@ defmodule Pleroma.Config.DeprecationWarnings do
Logger.warn(warning_preface <> warning) Logger.warn(warning_preface <> warning)
end end
end end
@spec check_media_proxy_whitelist_config() :: :ok | nil
def check_media_proxy_whitelist_config do
whitelist = Config.get([:media_proxy, :whitelist])
if Enum.any?(whitelist, &(not String.starts_with?(&1, "http"))) do
Logger.warn("""
!!!DEPRECATION WARNING!!!
Your config is using old format (only domain) for MediaProxy whitelist option. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later.
""")
end
end
end end

View file

@ -108,31 +108,48 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do
|> :erlang.iolist_to_binary() |> :erlang.iolist_to_binary()
end end
defp build_csp_from_whitelist([], acc), do: acc
defp build_csp_from_whitelist([last], acc) do
[build_csp_param_from_whitelist(last) | acc]
end
defp build_csp_from_whitelist([head | tail], acc) do
build_csp_from_whitelist(tail, [[?\s, build_csp_param_from_whitelist(head)] | acc])
end
# TODO: use `build_csp_param/1` after removing support bare domains for media proxy whitelist
defp build_csp_param_from_whitelist("http" <> _ = url) do
build_csp_param(url)
end
defp build_csp_param_from_whitelist(url), do: url
defp build_csp_multimedia_source_list do defp build_csp_multimedia_source_list do
media_proxy_whitelist = media_proxy_whitelist =
Enum.reduce(Config.get([:media_proxy, :whitelist]), [], fn host, acc -> [:media_proxy, :whitelist]
add_source(acc, host) |> Config.get()
end) |> build_csp_from_whitelist([])
media_proxy_base_url = build_csp_param(Config.get([:media_proxy, :base_url]))
upload_base_url = build_csp_param(Config.get([Pleroma.Upload, :base_url]))
s3_endpoint = build_csp_param(Config.get([Pleroma.Uploaders.S3, :public_endpoint]))
captcha_method = Config.get([Pleroma.Captcha, :method]) captcha_method = Config.get([Pleroma.Captcha, :method])
captcha_endpoint = Config.get([captcha_method, :endpoint])
captcha_endpoint = build_csp_param(Config.get([captcha_method, :endpoint])) base_endpoints =
[
[:media_proxy, :base_url],
[Pleroma.Upload, :base_url],
[Pleroma.Uploaders.S3, :public_endpoint]
]
|> Enum.map(&Config.get/1)
[] [captcha_endpoint | base_endpoints]
|> add_source(media_proxy_base_url) |> Enum.map(&build_csp_param/1)
|> add_source(upload_base_url) |> Enum.reduce([], &add_source(&2, &1))
|> add_source(s3_endpoint)
|> add_source(media_proxy_whitelist) |> add_source(media_proxy_whitelist)
|> add_source(captcha_endpoint)
end end
defp add_source(iodata, nil), do: iodata defp add_source(iodata, nil), do: iodata
defp add_source(iodata, []), do: iodata
defp add_source(iodata, source), do: [[?\s, source] | iodata] defp add_source(iodata, source), do: [[?\s, source] | iodata]
defp add_csp_param(csp_iodata, nil), do: csp_iodata defp add_csp_param(csp_iodata, nil), do: csp_iodata

View file

@ -60,22 +60,28 @@ defmodule Pleroma.Web.MediaProxy do
defp whitelisted?(url) do defp whitelisted?(url) do
%{host: domain} = URI.parse(url) %{host: domain} = URI.parse(url)
mediaproxy_whitelist = Config.get([:media_proxy, :whitelist]) mediaproxy_whitelist_domains =
[:media_proxy, :whitelist]
|> Config.get()
|> Enum.map(&maybe_get_domain_from_url/1)
upload_base_url_domain = whitelist_domains =
if !is_nil(Config.get([Upload, :base_url])) do if base_url = Config.get([Upload, :base_url]) do
[URI.parse(Config.get([Upload, :base_url])).host] %{host: base_domain} = URI.parse(base_url)
[base_domain | mediaproxy_whitelist_domains]
else else
[] mediaproxy_whitelist_domains
end end
whitelist = mediaproxy_whitelist ++ upload_base_url_domain domain in whitelist_domains
Enum.any?(whitelist, fn pattern ->
String.equivalent?(domain, pattern)
end)
end end
defp maybe_get_domain_from_url("http" <> _ = url) do
URI.parse(url).host
end
defp maybe_get_domain_from_url(domain), do: domain
def encode_url(url) do def encode_url(url) do
base64 = Base.url_encode64(url, @base64_opts) base64 = Base.url_encode64(url, @base64_opts)

View file

@ -54,4 +54,12 @@ defmodule Pleroma.Config.DeprecationWarningsTest do
assert Pleroma.Config.get(new_group2) == 2 assert Pleroma.Config.get(new_group2) == 2
assert Pleroma.Config.get(new_group3) == 3 assert Pleroma.Config.get(new_group3) == 3
end end
test "check_media_proxy_whitelist_config/0" do
clear_config([:media_proxy, :whitelist], ["https://example.com", "example2.com"])
assert capture_log(fn ->
Pleroma.Config.DeprecationWarnings.check_media_proxy_whitelist_config()
end) =~ "Your config is using old format (only domain) for MediaProxy whitelist option"
end
end end

View file

@ -4,17 +4,12 @@
defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do
use Pleroma.Web.ConnCase use Pleroma.Web.ConnCase
alias Pleroma.Config alias Pleroma.Config
alias Plug.Conn alias Plug.Conn
setup do: clear_config([:http_securiy, :enabled])
setup do: clear_config([:http_security, :sts])
setup do: clear_config([:http_security, :referrer_policy])
describe "http security enabled" do describe "http security enabled" do
setup do setup do: clear_config([:http_security, :enabled], true)
Config.put([:http_security, :enabled], true)
end
test "it sends CSP headers when enabled", %{conn: conn} do test "it sends CSP headers when enabled", %{conn: conn} do
conn = get(conn, "/api/v1/instance") conn = get(conn, "/api/v1/instance")
@ -29,7 +24,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do
end end
test "it sends STS headers when enabled", %{conn: conn} do test "it sends STS headers when enabled", %{conn: conn} do
Config.put([:http_security, :sts], true) clear_config([:http_security, :sts], true)
conn = get(conn, "/api/v1/instance") conn = get(conn, "/api/v1/instance")
@ -38,7 +33,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do
end end
test "it does not send STS headers when disabled", %{conn: conn} do test "it does not send STS headers when disabled", %{conn: conn} do
Config.put([:http_security, :sts], false) clear_config([:http_security, :sts], false)
conn = get(conn, "/api/v1/instance") conn = get(conn, "/api/v1/instance")
@ -47,23 +42,19 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do
end end
test "referrer-policy header reflects configured value", %{conn: conn} do test "referrer-policy header reflects configured value", %{conn: conn} do
conn = get(conn, "/api/v1/instance") resp = get(conn, "/api/v1/instance")
assert Conn.get_resp_header(conn, "referrer-policy") == ["same-origin"] assert Conn.get_resp_header(resp, "referrer-policy") == ["same-origin"]
Config.put([:http_security, :referrer_policy], "no-referrer") clear_config([:http_security, :referrer_policy], "no-referrer")
conn = resp = get(conn, "/api/v1/instance")
build_conn()
|> get("/api/v1/instance")
assert Conn.get_resp_header(conn, "referrer-policy") == ["no-referrer"] assert Conn.get_resp_header(resp, "referrer-policy") == ["no-referrer"]
end end
test "it sends `report-to` & `report-uri` CSP response headers" do test "it sends `report-to` & `report-uri` CSP response headers", %{conn: conn} do
conn = conn = get(conn, "/api/v1/instance")
build_conn()
|> get("/api/v1/instance")
[csp] = Conn.get_resp_header(conn, "content-security-policy") [csp] = Conn.get_resp_header(conn, "content-security-policy")
@ -74,10 +65,67 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do
assert reply_to == assert reply_to ==
"{\"endpoints\":[{\"url\":\"https://endpoint.com\"}],\"group\":\"csp-endpoint\",\"max-age\":10886400}" "{\"endpoints\":[{\"url\":\"https://endpoint.com\"}],\"group\":\"csp-endpoint\",\"max-age\":10886400}"
end end
test "default values for img-src and media-src with disabled media proxy", %{conn: conn} do
conn = get(conn, "/api/v1/instance")
[csp] = Conn.get_resp_header(conn, "content-security-policy")
assert csp =~ "media-src 'self' https:;"
assert csp =~ "img-src 'self' data: blob: https:;"
end
end
describe "img-src and media-src" do
setup do
clear_config([:http_security, :enabled], true)
clear_config([:media_proxy, :enabled], true)
clear_config([:media_proxy, :proxy_opts, :redirect_on_failure], false)
end
test "media_proxy with base_url", %{conn: conn} do
url = "https://example.com"
clear_config([:media_proxy, :base_url], url)
assert_media_img_src(conn, url)
end
test "upload with base url", %{conn: conn} do
url = "https://example2.com"
clear_config([Pleroma.Upload, :base_url], url)
assert_media_img_src(conn, url)
end
test "with S3 public endpoint", %{conn: conn} do
url = "https://example3.com"
clear_config([Pleroma.Uploaders.S3, :public_endpoint], url)
assert_media_img_src(conn, url)
end
test "with captcha endpoint", %{conn: conn} do
clear_config([Pleroma.Captcha.Mock, :endpoint], "https://captcha.com")
assert_media_img_src(conn, "https://captcha.com")
end
test "with media_proxy whitelist", %{conn: conn} do
clear_config([:media_proxy, :whitelist], ["https://example6.com", "https://example7.com"])
assert_media_img_src(conn, "https://example7.com https://example6.com")
end
# TODO: delete after removing support bare domains for media proxy whitelist
test "with media_proxy bare domains whitelist (deprecated)", %{conn: conn} do
clear_config([:media_proxy, :whitelist], ["example4.com", "example5.com"])
assert_media_img_src(conn, "example5.com example4.com")
end
end
defp assert_media_img_src(conn, url) do
conn = get(conn, "/api/v1/instance")
[csp] = Conn.get_resp_header(conn, "content-security-policy")
assert csp =~ "media-src 'self' #{url};"
assert csp =~ "img-src 'self' data: blob: #{url};"
end end
test "it does not send CSP headers when disabled", %{conn: conn} do test "it does not send CSP headers when disabled", %{conn: conn} do
Config.put([:http_security, :enabled], false) clear_config([:http_security, :enabled], false)
conn = get(conn, "/api/v1/instance") conn = get(conn, "/api/v1/instance")

View file

@ -4,82 +4,118 @@
defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do
use Pleroma.Web.ConnCase use Pleroma.Web.ConnCase
import Mock
alias Pleroma.Config
setup do: clear_config(:media_proxy) import Mock
setup do: clear_config([Pleroma.Web.Endpoint, :secret_key_base])
alias Pleroma.Web.MediaProxy
alias Pleroma.Web.MediaProxy.MediaProxyController
alias Plug.Conn
setup do setup do
on_exit(fn -> Cachex.clear(:banned_urls_cache) end) on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
end end
test "it returns 404 when MediaProxy disabled", %{conn: conn} do test "it returns 404 when MediaProxy disabled", %{conn: conn} do
Config.put([:media_proxy, :enabled], false) clear_config([:media_proxy, :enabled], false)
assert %Plug.Conn{ assert %Conn{
status: 404, status: 404,
resp_body: "Not Found" resp_body: "Not Found"
} = get(conn, "/proxy/hhgfh/eeeee") } = get(conn, "/proxy/hhgfh/eeeee")
assert %Plug.Conn{ assert %Conn{
status: 404, status: 404,
resp_body: "Not Found" resp_body: "Not Found"
} = get(conn, "/proxy/hhgfh/eeee/fff") } = get(conn, "/proxy/hhgfh/eeee/fff")
end end
test "it returns 403 when signature invalidated", %{conn: conn} do describe "" do
Config.put([:media_proxy, :enabled], true) setup do
Config.put([Pleroma.Web.Endpoint, :secret_key_base], "00000000000") clear_config([:media_proxy, :enabled], true)
path = URI.parse(Pleroma.Web.MediaProxy.encode_url("https://google.fn")).path clear_config([Pleroma.Web.Endpoint, :secret_key_base], "00000000000")
Config.put([Pleroma.Web.Endpoint, :secret_key_base], "000") [url: MediaProxy.encode_url("https://google.fn/test.png")]
end
assert %Plug.Conn{ test "it returns 403 for invalid signature", %{conn: conn, url: url} do
status: 403, Pleroma.Config.put([Pleroma.Web.Endpoint, :secret_key_base], "000")
resp_body: "Forbidden" %{path: path} = URI.parse(url)
} = get(conn, path)
assert %Plug.Conn{ assert %Conn{
status: 403, status: 403,
resp_body: "Forbidden" resp_body: "Forbidden"
} = get(conn, "/proxy/hhgfh/eeee") } = get(conn, path)
assert %Plug.Conn{ assert %Conn{
status: 403, status: 403,
resp_body: "Forbidden" resp_body: "Forbidden"
} = get(conn, "/proxy/hhgfh/eeee/fff") } = get(conn, "/proxy/hhgfh/eeee")
end
test "redirects on valid url when filename invalidated", %{conn: conn} do assert %Conn{
Config.put([:media_proxy, :enabled], true) status: 403,
Config.put([Pleroma.Web.Endpoint, :secret_key_base], "00000000000") resp_body: "Forbidden"
url = Pleroma.Web.MediaProxy.encode_url("https://google.fn/test.png") } = get(conn, "/proxy/hhgfh/eeee/fff")
invalid_url = String.replace(url, "test.png", "test-file.png") end
response = get(conn, invalid_url)
assert response.status == 302
assert redirected_to(response) == url
end
test "it performs ReverseProxy.call when signature valid", %{conn: conn} do test "redirects on valid url when filename is invalidated", %{conn: conn, url: url} do
Config.put([:media_proxy, :enabled], true) invalid_url = String.replace(url, "test.png", "test-file.png")
Config.put([Pleroma.Web.Endpoint, :secret_key_base], "00000000000") response = get(conn, invalid_url)
url = Pleroma.Web.MediaProxy.encode_url("https://google.fn/test.png") assert response.status == 302
assert redirected_to(response) == url
end
with_mock Pleroma.ReverseProxy, test "it performs ReverseProxy.call with valid signature", %{conn: conn, url: url} do
call: fn _conn, _url, _opts -> %Plug.Conn{status: :success} end do with_mock Pleroma.ReverseProxy,
assert %Plug.Conn{status: :success} = get(conn, url) call: fn _conn, _url, _opts -> %Conn{status: :success} end do
assert %Conn{status: :success} = get(conn, url)
end
end
test "it returns 404 when url is in banned_urls cache", %{conn: conn, url: url} do
MediaProxy.put_in_banned_urls("https://google.fn/test.png")
with_mock Pleroma.ReverseProxy,
call: fn _conn, _url, _opts -> %Conn{status: :success} end do
assert %Conn{status: 404, resp_body: "Not Found"} = get(conn, url)
end
end end
end end
test "it returns 404 when url contains in banned_urls cache", %{conn: conn} do describe "filename_matches/3" do
Config.put([:media_proxy, :enabled], true) test "preserves the encoded or decoded path" do
Config.put([Pleroma.Web.Endpoint, :secret_key_base], "00000000000") assert MediaProxyController.filename_matches(
url = Pleroma.Web.MediaProxy.encode_url("https://google.fn/test.png") %{"filename" => "/Hello world.jpg"},
Pleroma.Web.MediaProxy.put_in_banned_urls("https://google.fn/test.png") "/Hello world.jpg",
"http://pleroma.social/Hello world.jpg"
) == :ok
with_mock Pleroma.ReverseProxy, assert MediaProxyController.filename_matches(
call: fn _conn, _url, _opts -> %Plug.Conn{status: :success} end do %{"filename" => "/Hello%20world.jpg"},
assert %Plug.Conn{status: 404, resp_body: "Not Found"} = get(conn, url) "/Hello%20world.jpg",
"http://pleroma.social/Hello%20world.jpg"
) == :ok
assert MediaProxyController.filename_matches(
%{"filename" => "/my%2Flong%2Furl%2F2019%2F07%2FS.jpg"},
"/my%2Flong%2Furl%2F2019%2F07%2FS.jpg",
"http://pleroma.social/my%2Flong%2Furl%2F2019%2F07%2FS.jpg"
) == :ok
assert MediaProxyController.filename_matches(
%{"filename" => "/my%2Flong%2Furl%2F2019%2F07%2FS.jp"},
"/my%2Flong%2Furl%2F2019%2F07%2FS.jp",
"http://pleroma.social/my%2Flong%2Furl%2F2019%2F07%2FS.jpg"
) == {:wrong_filename, "my%2Flong%2Furl%2F2019%2F07%2FS.jpg"}
end
test "encoded url are tried to match for proxy as `conn.request_path` encodes the url" do
# conn.request_path will return encoded url
request_path = "/ANALYSE-DAI-_-LE-STABLECOIN-100-D%C3%89CENTRALIS%C3%89-BQ.jpg"
assert MediaProxyController.filename_matches(
true,
request_path,
"https://mydomain.com/uploads/2019/07/ANALYSE-DAI-_-LE-STABLECOIN-100-DÉCENTRALISÉ-BQ.jpg"
) == :ok
end end
end end
end end

View file

@ -5,38 +5,33 @@
defmodule Pleroma.Web.MediaProxyTest do defmodule Pleroma.Web.MediaProxyTest do
use ExUnit.Case use ExUnit.Case
use Pleroma.Tests.Helpers use Pleroma.Tests.Helpers
import Pleroma.Web.MediaProxy
alias Pleroma.Web.MediaProxy.MediaProxyController
setup do: clear_config([:media_proxy, :enabled]) alias Pleroma.Web.Endpoint
setup do: clear_config(Pleroma.Upload) alias Pleroma.Web.MediaProxy
describe "when enabled" do describe "when enabled" do
setup do setup do: clear_config([:media_proxy, :enabled], true)
Pleroma.Config.put([:media_proxy, :enabled], true)
:ok
end
test "ignores invalid url" do test "ignores invalid url" do
assert url(nil) == nil assert MediaProxy.url(nil) == nil
assert url("") == nil assert MediaProxy.url("") == nil
end end
test "ignores relative url" do test "ignores relative url" do
assert url("/local") == "/local" assert MediaProxy.url("/local") == "/local"
assert url("/") == "/" assert MediaProxy.url("/") == "/"
end end
test "ignores local url" do test "ignores local url" do
local_url = Pleroma.Web.Endpoint.url() <> "/hello" local_url = Endpoint.url() <> "/hello"
local_root = Pleroma.Web.Endpoint.url() local_root = Endpoint.url()
assert url(local_url) == local_url assert MediaProxy.url(local_url) == local_url
assert url(local_root) == local_root assert MediaProxy.url(local_root) == local_root
end end
test "encodes and decodes URL" do test "encodes and decodes URL" do
url = "https://pleroma.soykaf.com/static/logo.png" url = "https://pleroma.soykaf.com/static/logo.png"
encoded = url(url) encoded = MediaProxy.url(url)
assert String.starts_with?( assert String.starts_with?(
encoded, encoded,
@ -50,86 +45,44 @@ defmodule Pleroma.Web.MediaProxyTest do
test "encodes and decodes URL without a path" do test "encodes and decodes URL without a path" do
url = "https://pleroma.soykaf.com" url = "https://pleroma.soykaf.com"
encoded = url(url) encoded = MediaProxy.url(url)
assert decode_result(encoded) == url assert decode_result(encoded) == url
end end
test "encodes and decodes URL without an extension" do test "encodes and decodes URL without an extension" do
url = "https://pleroma.soykaf.com/path/" url = "https://pleroma.soykaf.com/path/"
encoded = url(url) encoded = MediaProxy.url(url)
assert String.ends_with?(encoded, "/path") assert String.ends_with?(encoded, "/path")
assert decode_result(encoded) == url assert decode_result(encoded) == url
end end
test "encodes and decodes URL and ignores query params for the path" do test "encodes and decodes URL and ignores query params for the path" do
url = "https://pleroma.soykaf.com/static/logo.png?93939393939&bunny=true" url = "https://pleroma.soykaf.com/static/logo.png?93939393939&bunny=true"
encoded = url(url) encoded = MediaProxy.url(url)
assert String.ends_with?(encoded, "/logo.png") assert String.ends_with?(encoded, "/logo.png")
assert decode_result(encoded) == url assert decode_result(encoded) == url
end end
test "validates signature" do test "validates signature" do
secret_key_base = Pleroma.Config.get([Pleroma.Web.Endpoint, :secret_key_base]) encoded = MediaProxy.url("https://pleroma.social")
on_exit(fn -> clear_config(
Pleroma.Config.put([Pleroma.Web.Endpoint, :secret_key_base], secret_key_base) [Endpoint, :secret_key_base],
end)
encoded = url("https://pleroma.social")
Pleroma.Config.put(
[Pleroma.Web.Endpoint, :secret_key_base],
"00000000000000000000000000000000000000000000000" "00000000000000000000000000000000000000000000000"
) )
[_, "proxy", sig, base64 | _] = URI.parse(encoded).path |> String.split("/") [_, "proxy", sig, base64 | _] = URI.parse(encoded).path |> String.split("/")
assert decode_url(sig, base64) == {:error, :invalid_signature} assert MediaProxy.decode_url(sig, base64) == {:error, :invalid_signature}
end
test "filename_matches preserves the encoded or decoded path" do
assert MediaProxyController.filename_matches(
%{"filename" => "/Hello world.jpg"},
"/Hello world.jpg",
"http://pleroma.social/Hello world.jpg"
) == :ok
assert MediaProxyController.filename_matches(
%{"filename" => "/Hello%20world.jpg"},
"/Hello%20world.jpg",
"http://pleroma.social/Hello%20world.jpg"
) == :ok
assert MediaProxyController.filename_matches(
%{"filename" => "/my%2Flong%2Furl%2F2019%2F07%2FS.jpg"},
"/my%2Flong%2Furl%2F2019%2F07%2FS.jpg",
"http://pleroma.social/my%2Flong%2Furl%2F2019%2F07%2FS.jpg"
) == :ok
assert MediaProxyController.filename_matches(
%{"filename" => "/my%2Flong%2Furl%2F2019%2F07%2FS.jp"},
"/my%2Flong%2Furl%2F2019%2F07%2FS.jp",
"http://pleroma.social/my%2Flong%2Furl%2F2019%2F07%2FS.jpg"
) == {:wrong_filename, "my%2Flong%2Furl%2F2019%2F07%2FS.jpg"}
end
test "encoded url are tried to match for proxy as `conn.request_path` encodes the url" do
# conn.request_path will return encoded url
request_path = "/ANALYSE-DAI-_-LE-STABLECOIN-100-D%C3%89CENTRALIS%C3%89-BQ.jpg"
assert MediaProxyController.filename_matches(
true,
request_path,
"https://mydomain.com/uploads/2019/07/ANALYSE-DAI-_-LE-STABLECOIN-100-DÉCENTRALISÉ-BQ.jpg"
) == :ok
end end
test "uses the configured base_url" do test "uses the configured base_url" do
clear_config([:media_proxy, :base_url], "https://cache.pleroma.social") base_url = "https://cache.pleroma.social"
clear_config([:media_proxy, :base_url], base_url)
url = "https://pleroma.soykaf.com/static/logo.png" url = "https://pleroma.soykaf.com/static/logo.png"
encoded = url(url) encoded = MediaProxy.url(url)
assert String.starts_with?(encoded, Pleroma.Config.get([:media_proxy, :base_url])) assert String.starts_with?(encoded, base_url)
end end
# Some sites expect ASCII encoded characters in the URL to be preserved even if # Some sites expect ASCII encoded characters in the URL to be preserved even if
@ -140,7 +93,7 @@ defmodule Pleroma.Web.MediaProxyTest do
url = url =
"https://pleroma.com/%20/%21/%22/%23/%24/%25/%26/%27/%28/%29/%2A/%2B/%2C/%2D/%2E/%2F/%30/%31/%32/%33/%34/%35/%36/%37/%38/%39/%3A/%3B/%3C/%3D/%3E/%3F/%40/%41/%42/%43/%44/%45/%46/%47/%48/%49/%4A/%4B/%4C/%4D/%4E/%4F/%50/%51/%52/%53/%54/%55/%56/%57/%58/%59/%5A/%5B/%5C/%5D/%5E/%5F/%60/%61/%62/%63/%64/%65/%66/%67/%68/%69/%6A/%6B/%6C/%6D/%6E/%6F/%70/%71/%72/%73/%74/%75/%76/%77/%78/%79/%7A/%7B/%7C/%7D/%7E/%7F/%80/%81/%82/%83/%84/%85/%86/%87/%88/%89/%8A/%8B/%8C/%8D/%8E/%8F/%90/%91/%92/%93/%94/%95/%96/%97/%98/%99/%9A/%9B/%9C/%9D/%9E/%9F/%C2%A0/%A1/%A2/%A3/%A4/%A5/%A6/%A7/%A8/%A9/%AA/%AB/%AC/%C2%AD/%AE/%AF/%B0/%B1/%B2/%B3/%B4/%B5/%B6/%B7/%B8/%B9/%BA/%BB/%BC/%BD/%BE/%BF/%C0/%C1/%C2/%C3/%C4/%C5/%C6/%C7/%C8/%C9/%CA/%CB/%CC/%CD/%CE/%CF/%D0/%D1/%D2/%D3/%D4/%D5/%D6/%D7/%D8/%D9/%DA/%DB/%DC/%DD/%DE/%DF/%E0/%E1/%E2/%E3/%E4/%E5/%E6/%E7/%E8/%E9/%EA/%EB/%EC/%ED/%EE/%EF/%F0/%F1/%F2/%F3/%F4/%F5/%F6/%F7/%F8/%F9/%FA/%FB/%FC/%FD/%FE/%FF" "https://pleroma.com/%20/%21/%22/%23/%24/%25/%26/%27/%28/%29/%2A/%2B/%2C/%2D/%2E/%2F/%30/%31/%32/%33/%34/%35/%36/%37/%38/%39/%3A/%3B/%3C/%3D/%3E/%3F/%40/%41/%42/%43/%44/%45/%46/%47/%48/%49/%4A/%4B/%4C/%4D/%4E/%4F/%50/%51/%52/%53/%54/%55/%56/%57/%58/%59/%5A/%5B/%5C/%5D/%5E/%5F/%60/%61/%62/%63/%64/%65/%66/%67/%68/%69/%6A/%6B/%6C/%6D/%6E/%6F/%70/%71/%72/%73/%74/%75/%76/%77/%78/%79/%7A/%7B/%7C/%7D/%7E/%7F/%80/%81/%82/%83/%84/%85/%86/%87/%88/%89/%8A/%8B/%8C/%8D/%8E/%8F/%90/%91/%92/%93/%94/%95/%96/%97/%98/%99/%9A/%9B/%9C/%9D/%9E/%9F/%C2%A0/%A1/%A2/%A3/%A4/%A5/%A6/%A7/%A8/%A9/%AA/%AB/%AC/%C2%AD/%AE/%AF/%B0/%B1/%B2/%B3/%B4/%B5/%B6/%B7/%B8/%B9/%BA/%BB/%BC/%BD/%BE/%BF/%C0/%C1/%C2/%C3/%C4/%C5/%C6/%C7/%C8/%C9/%CA/%CB/%CC/%CD/%CE/%CF/%D0/%D1/%D2/%D3/%D4/%D5/%D6/%D7/%D8/%D9/%DA/%DB/%DC/%DD/%DE/%DF/%E0/%E1/%E2/%E3/%E4/%E5/%E6/%E7/%E8/%E9/%EA/%EB/%EC/%ED/%EE/%EF/%F0/%F1/%F2/%F3/%F4/%F5/%F6/%F7/%F8/%F9/%FA/%FB/%FC/%FD/%FE/%FF"
encoded = url(url) encoded = MediaProxy.url(url)
assert decode_result(encoded) == url assert decode_result(encoded) == url
end end
@ -151,56 +104,49 @@ defmodule Pleroma.Web.MediaProxyTest do
url = url =
"https://pleroma.com/ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890-._~:/?#[]@!$&'()*+,;=|^`{}" "https://pleroma.com/ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890-._~:/?#[]@!$&'()*+,;=|^`{}"
encoded = url(url) encoded = MediaProxy.url(url)
assert decode_result(encoded) == url assert decode_result(encoded) == url
end end
test "preserve unicode characters" do test "preserve unicode characters" do
url = "https://ko.wikipedia.org/wiki/위키백과:대문" url = "https://ko.wikipedia.org/wiki/위키백과:대문"
encoded = url(url) encoded = MediaProxy.url(url)
assert decode_result(encoded) == url assert decode_result(encoded) == url
end end
end end
describe "when disabled" do describe "when disabled" do
setup do setup do: clear_config([:media_proxy, :enabled], false)
enabled = Pleroma.Config.get([:media_proxy, :enabled])
if enabled do
Pleroma.Config.put([:media_proxy, :enabled], false)
on_exit(fn ->
Pleroma.Config.put([:media_proxy, :enabled], enabled)
:ok
end)
end
:ok
end
test "does not encode remote urls" do test "does not encode remote urls" do
assert url("https://google.fr") == "https://google.fr" assert MediaProxy.url("https://google.fr") == "https://google.fr"
end end
end end
defp decode_result(encoded) do defp decode_result(encoded) do
[_, "proxy", sig, base64 | _] = URI.parse(encoded).path |> String.split("/") [_, "proxy", sig, base64 | _] = URI.parse(encoded).path |> String.split("/")
{:ok, decoded} = decode_url(sig, base64) {:ok, decoded} = MediaProxy.decode_url(sig, base64)
decoded decoded
end end
describe "whitelist" do describe "whitelist" do
setup do setup do: clear_config([:media_proxy, :enabled], true)
Pleroma.Config.put([:media_proxy, :enabled], true)
:ok
end
test "mediaproxy whitelist" do test "mediaproxy whitelist" do
Pleroma.Config.put([:media_proxy, :whitelist], ["google.com", "feld.me"]) clear_config([:media_proxy, :whitelist], ["https://google.com", "https://feld.me"])
url = "https://feld.me/foo.png" url = "https://feld.me/foo.png"
unencoded = url(url) unencoded = MediaProxy.url(url)
assert unencoded == url
end
# TODO: delete after removing support bare domains for media proxy whitelist
test "mediaproxy whitelist bare domains whitelist (deprecated)" do
clear_config([:media_proxy, :whitelist], ["google.com", "feld.me"])
url = "https://feld.me/foo.png"
unencoded = MediaProxy.url(url)
assert unencoded == url assert unencoded == url
end end
@ -211,17 +157,17 @@ defmodule Pleroma.Web.MediaProxyTest do
media_url = "https://mycdn.akamai.com" media_url = "https://mycdn.akamai.com"
url = "#{media_url}/static/logo.png" url = "#{media_url}/static/logo.png"
encoded = url(url) encoded = MediaProxy.url(url)
assert String.starts_with?(encoded, media_url) assert String.starts_with?(encoded, media_url)
end end
test "ensure Pleroma.Upload base_url is always whitelisted" do test "ensure Pleroma.Upload base_url is always whitelisted" do
media_url = "https://media.pleroma.social" media_url = "https://media.pleroma.social"
Pleroma.Config.put([Pleroma.Upload, :base_url], media_url) clear_config([Pleroma.Upload, :base_url], media_url)
url = "#{media_url}/static/logo.png" url = "#{media_url}/static/logo.png"
encoded = url(url) encoded = MediaProxy.url(url)
assert String.starts_with?(encoded, media_url) assert String.starts_with?(encoded, media_url)
end end