From dee4639dbb1245d4514b7b81d567321f9b4ee099 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 14 Sep 2020 12:38:00 +0000 Subject: [PATCH] Merge branch 'feat/rich-media-head' into 'develop' RichMedia: Do a HEAD request to check content type/length See merge request pleroma/pleroma!2995 --- lib/pleroma/web/rich_media/helpers.ex | 46 ++++++++++++++++++++++++++- lib/pleroma/web/rich_media/parser.ex | 8 +++++ test/support/http_request_mock.ex | 17 ++++++++++ test/web/rich_media/parser_test.exs | 29 +++++++++++++++++ 4 files changed, 99 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 752ca9f81..b7852c6e3 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -96,6 +96,50 @@ defmodule Pleroma.Web.RichMedia.Helpers do @rich_media_options end - Pleroma.HTTP.get(url, headers, adapter: options) + head_check = + case Pleroma.HTTP.head(url, headers, adapter: options) do + # If the HEAD request didn't reach the server for whatever reason, + # we assume the GET that comes right after won't either + {:error, _} = e -> + e + + {:ok, %Tesla.Env{status: 200, headers: headers}} -> + with :ok <- check_content_type(headers), + :ok <- check_content_length(headers), + do: :ok + + _ -> + :ok + end + + with :ok <- head_check, do: Pleroma.HTTP.get(url, headers, adapter: options) + end + + defp check_content_type(headers) do + case List.keyfind(headers, "content-type", 0) do + {_, content_type} -> + case Plug.Conn.Utils.media_type(content_type) do + {:ok, "text", "html", _} -> :ok + _ -> {:error, {:content_type, content_type}} + end + + _ -> + :ok + end + end + + @max_body @rich_media_options[:max_body] + defp check_content_length(headers) do + case List.keyfind(headers, "content-length", 0) do + {_, maybe_content_length} -> + case Integer.parse(maybe_content_length) do + {content_length, ""} when content_length <= @max_body -> :ok + {_, ""} -> {:error, :body_too_large} + _ -> :ok + end + + _ -> + :ok + end end end diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index e98c743ca..49ba22c90 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -31,6 +31,14 @@ defmodule Pleroma.Web.RichMedia.Parser do {:ok, _data} = res -> res + {:error, :body_too_large} = e -> + e + + {:error, {:content_type, _}} -> + e + + # The TTL is not set for the errors above, since they are unlikely to change + # with time {:error, _} = e -> ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000) Cachex.expire(:rich_media_cache, url, ttl) diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index a0ebf65d9..d9be248dc 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -1436,4 +1436,21 @@ defmodule HttpRequestMock do inspect(headers) }"} end + + # Most of the rich media mocks are missing HEAD requests, so we just return 404. + @rich_media_mocks [ + "https://example.com/ogp", + "https://example.com/ogp-missing-data", + "https://example.com/twitter-card" + ] + def head(url, _query, _body, _headers) when url in @rich_media_mocks do + {:ok, %Tesla.Env{status: 404, body: ""}} + end + + def head(url, query, body, headers) do + {:error, + "Mock response not implemented for HEAD #{inspect(url)}, #{query}, #{inspect(body)}, #{ + inspect(headers) + }"} + end end diff --git a/test/web/rich_media/parser_test.exs b/test/web/rich_media/parser_test.exs index 1e09cbf84..b8ef2cccf 100644 --- a/test/web/rich_media/parser_test.exs +++ b/test/web/rich_media/parser_test.exs @@ -56,6 +56,27 @@ defmodule Pleroma.Web.RichMedia.ParserTest do %{method: :get, url: "http://example.com/error"} -> {:error, :overload} + + %{ + method: :head, + url: "http://example.com/huge-page" + } -> + %Tesla.Env{ + status: 200, + headers: [{"content-length", "2000001"}, {"content-type", "text/html"}] + } + + %{ + method: :head, + url: "http://example.com/pdf-file" + } -> + %Tesla.Env{ + status: 200, + headers: [{"content-length", "1000000"}, {"content-type", "application/pdf"}] + } + + %{method: :head} -> + %Tesla.Env{status: 404, body: "", headers: []} end) :ok @@ -146,4 +167,12 @@ defmodule Pleroma.Web.RichMedia.ParserTest do test "returns error if getting page was not successful" do assert {:error, :overload} = Parser.parse("http://example.com/error") end + + test "does a HEAD request to check if the body is too large" do + assert {:error, :body_too_large} = Parser.parse("http://example.com/huge-page") + end + + test "does a HEAD request to check if the body is html" do + assert {:error, {:content_type, _}} = Parser.parse("http://example.com/pdf-file") + end end