forked from AkkomaGang/akkoma
Merge branch 'feat/rich-media-head' into 'develop'
RichMedia: Do a HEAD request to check content type/length See merge request pleroma/pleroma!2995
This commit is contained in:
commit
4d543fcb75
4 changed files with 99 additions and 1 deletions
lib/pleroma/web/rich_media
test
|
@ -87,6 +87,50 @@ def perform(:fetch, %Activity{} = activity) do
|
||||||
def rich_media_get(url) do
|
def rich_media_get(url) do
|
||||||
headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}]
|
headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}]
|
||||||
|
|
||||||
Pleroma.HTTP.get(url, headers, @options)
|
head_check =
|
||||||
|
case Pleroma.HTTP.head(url, headers, @options) do
|
||||||
|
# If the HEAD request didn't reach the server for whatever reason,
|
||||||
|
# we assume the GET that comes right after won't either
|
||||||
|
{:error, _} = e ->
|
||||||
|
e
|
||||||
|
|
||||||
|
{:ok, %Tesla.Env{status: 200, headers: headers}} ->
|
||||||
|
with :ok <- check_content_type(headers),
|
||||||
|
:ok <- check_content_length(headers),
|
||||||
|
do: :ok
|
||||||
|
|
||||||
|
_ ->
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
|
||||||
|
with :ok <- head_check, do: Pleroma.HTTP.get(url, headers, @options)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp check_content_type(headers) do
|
||||||
|
case List.keyfind(headers, "content-type", 0) do
|
||||||
|
{_, content_type} ->
|
||||||
|
case Plug.Conn.Utils.media_type(content_type) do
|
||||||
|
{:ok, "text", "html", _} -> :ok
|
||||||
|
_ -> {:error, {:content_type, content_type}}
|
||||||
|
end
|
||||||
|
|
||||||
|
_ ->
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
@max_body @options[:max_body]
|
||||||
|
defp check_content_length(headers) do
|
||||||
|
case List.keyfind(headers, "content-length", 0) do
|
||||||
|
{_, maybe_content_length} ->
|
||||||
|
case Integer.parse(maybe_content_length) do
|
||||||
|
{content_length, ""} when content_length <= @max_body -> :ok
|
||||||
|
{_, ""} -> {:error, :body_too_large}
|
||||||
|
_ -> :ok
|
||||||
|
end
|
||||||
|
|
||||||
|
_ ->
|
||||||
|
:ok
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -36,6 +36,14 @@ defp get_cached_or_parse(url) do
|
||||||
{:ok, _data} = res ->
|
{:ok, _data} = res ->
|
||||||
res
|
res
|
||||||
|
|
||||||
|
{:error, :body_too_large} = e ->
|
||||||
|
e
|
||||||
|
|
||||||
|
{:error, {:content_type, _}} ->
|
||||||
|
e
|
||||||
|
|
||||||
|
# The TTL is not set for the errors above, since they are unlikely to change
|
||||||
|
# with time
|
||||||
{:error, _} = e ->
|
{:error, _} = e ->
|
||||||
ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000)
|
ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000)
|
||||||
Cachex.expire(:rich_media_cache, url, ttl)
|
Cachex.expire(:rich_media_cache, url, ttl)
|
||||||
|
|
|
@ -1262,4 +1262,21 @@ def post(url, query, body, headers) do
|
||||||
inspect(headers)
|
inspect(headers)
|
||||||
}"}
|
}"}
|
||||||
end
|
end
|
||||||
|
|
||||||
|
# Most of the rich media mocks are missing HEAD requests, so we just return 404.
|
||||||
|
@rich_media_mocks [
|
||||||
|
"https://example.com/ogp",
|
||||||
|
"https://example.com/ogp-missing-data",
|
||||||
|
"https://example.com/twitter-card"
|
||||||
|
]
|
||||||
|
def head(url, _query, _body, _headers) when url in @rich_media_mocks do
|
||||||
|
{:ok, %Tesla.Env{status: 404, body: ""}}
|
||||||
|
end
|
||||||
|
|
||||||
|
def head(url, query, body, headers) do
|
||||||
|
{:error,
|
||||||
|
"Mock response not implemented for HEAD #{inspect(url)}, #{query}, #{inspect(body)}, #{
|
||||||
|
inspect(headers)
|
||||||
|
}"}
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -56,6 +56,27 @@ defmodule Pleroma.Web.RichMedia.ParserTest do
|
||||||
|
|
||||||
%{method: :get, url: "http://example.com/error"} ->
|
%{method: :get, url: "http://example.com/error"} ->
|
||||||
{:error, :overload}
|
{:error, :overload}
|
||||||
|
|
||||||
|
%{
|
||||||
|
method: :head,
|
||||||
|
url: "http://example.com/huge-page"
|
||||||
|
} ->
|
||||||
|
%Tesla.Env{
|
||||||
|
status: 200,
|
||||||
|
headers: [{"content-length", "2000001"}, {"content-type", "text/html"}]
|
||||||
|
}
|
||||||
|
|
||||||
|
%{
|
||||||
|
method: :head,
|
||||||
|
url: "http://example.com/pdf-file"
|
||||||
|
} ->
|
||||||
|
%Tesla.Env{
|
||||||
|
status: 200,
|
||||||
|
headers: [{"content-length", "1000000"}, {"content-type", "application/pdf"}]
|
||||||
|
}
|
||||||
|
|
||||||
|
%{method: :head} ->
|
||||||
|
%Tesla.Env{status: 404, body: "", headers: []}
|
||||||
end)
|
end)
|
||||||
|
|
||||||
:ok
|
:ok
|
||||||
|
@ -144,4 +165,12 @@ test "rejects invalid OGP data" do
|
||||||
test "returns error if getting page was not successful" do
|
test "returns error if getting page was not successful" do
|
||||||
assert {:error, :overload} = Parser.parse("http://example.com/error")
|
assert {:error, :overload} = Parser.parse("http://example.com/error")
|
||||||
end
|
end
|
||||||
|
|
||||||
|
test "does a HEAD request to check if the body is too large" do
|
||||||
|
assert {:error, :body_too_large} = Parser.parse("http://example.com/huge-page")
|
||||||
|
end
|
||||||
|
|
||||||
|
test "does a HEAD request to check if the body is html" do
|
||||||
|
assert {:error, {:content_type, _}} = Parser.parse("http://example.com/pdf-file")
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
Loading…
Reference in a new issue