adapter options unification

not needed options deletion
This commit is contained in:
Alexander Strizhakov 2020-09-04 19:05:08 +03:00
parent c5434dbefc
commit a83916fdac
No known key found for this signature in database
GPG key ID: 022896A53AEF1381
11 changed files with 47 additions and 63 deletions

View file

@ -735,28 +735,28 @@ config :pleroma, :connections_pool,
max_connections: 250, max_connections: 250,
max_idle_time: 30_000, max_idle_time: 30_000,
retry: 0, retry: 0,
await_up_timeout: 5_000 connect_timeout: 5_000
config :pleroma, :pools, config :pleroma, :pools,
federation: [ federation: [
size: 50, size: 50,
max_waiting: 10, max_waiting: 10,
timeout: 10_000 recv_timeout: 10_000
], ],
media: [ media: [
size: 50, size: 50,
max_waiting: 10, max_waiting: 10,
timeout: 10_000 recv_timeout: 10_000
], ],
upload: [ upload: [
size: 25, size: 25,
max_waiting: 5, max_waiting: 5,
timeout: 15_000 recv_timeout: 15_000
], ],
default: [ default: [
size: 10, size: 10,
max_waiting: 2, max_waiting: 2,
timeout: 5_000 recv_timeout: 5_000
] ]
config :pleroma, :hackney_pools, config :pleroma, :hackney_pools,

View file

@ -3377,7 +3377,7 @@ config :pleroma, :config_description, [
suggestions: [250] suggestions: [250]
}, },
%{ %{
key: :await_up_timeout, key: :connect_timeout,
type: :integer, type: :integer,
description: "Timeout while `gun` will wait until connection is up. Default: 5000ms.", description: "Timeout while `gun` will wait until connection is up. Default: 5000ms.",
suggestions: [5000] suggestions: [5000]
@ -3415,6 +3415,12 @@ config :pleroma, :config_description, [
description: description:
"Maximum number of requests waiting for other requests to finish. After this number is reached, the pool will start returning errrors when a new request is made", "Maximum number of requests waiting for other requests to finish. After this number is reached, the pool will start returning errrors when a new request is made",
suggestions: [10] suggestions: [10]
},
%{
key: :recv_timeout,
type: :integer,
description: "Timeout for the pool while gun will wait for response",
suggestions: [10_000]
} }
] ]
} }

View file

@ -498,7 +498,7 @@ Settings for HTTP connection pool.
* `:connection_acquisition_wait` - Timeout to acquire a connection from pool.The total max time is this value multiplied by the number of retries. * `:connection_acquisition_wait` - Timeout to acquire a connection from pool.The total max time is this value multiplied by the number of retries.
* `connection_acquisition_retries` - Number of attempts to acquire the connection from the pool if it is overloaded. Each attempt is timed `:connection_acquisition_wait` apart. * `connection_acquisition_retries` - Number of attempts to acquire the connection from the pool if it is overloaded. Each attempt is timed `:connection_acquisition_wait` apart.
* `:max_connections` - Maximum number of connections in the pool. * `:max_connections` - Maximum number of connections in the pool.
* `:await_up_timeout` - Timeout to connect to the host. * `:connect_timeout` - Timeout to connect to the host.
* `:reclaim_multiplier` - Multiplied by `:max_connections` this will be the maximum number of idle connections that will be reclaimed in case the pool is overloaded. * `:reclaim_multiplier` - Multiplied by `:max_connections` this will be the maximum number of idle connections that will be reclaimed in case the pool is overloaded.
### :pools ### :pools
@ -517,7 +517,7 @@ There are four pools used:
For each pool, the options are: For each pool, the options are:
* `:size` - limit to how much requests can be concurrently executed. * `:size` - limit to how much requests can be concurrently executed.
* `:timeout` - timeout while `gun` will wait for response * `:recv_timeout` - timeout while `gun` will wait for response
* `:max_waiting` - limit to how much requests can be waiting for others to finish, after this is reached, subsequent requests will be dropped. * `:max_waiting` - limit to how much requests can be waiting for others to finish, after this is reached, subsequent requests will be dropped.
## Captcha ## Captcha

View file

@ -124,9 +124,7 @@ defmodule Mix.Tasks.Pleroma.Frontend do
url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"]) url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
with {:ok, %{status: 200, body: zip_body}} <- with {:ok, %{status: 200, body: zip_body}} <-
Pleroma.HTTP.get(url, [], Pleroma.HTTP.get(url, [], adapter: [pool: :media, recv_timeout: 120_000]) do
adapter: [pool: :media, timeout: 120_000, recv_timeout: 120_000]
) do
unzip(zip_body, dest) unzip(zip_body, dest)
else else
e -> {:error, e} e -> {:error, e}

View file

@ -13,7 +13,7 @@ defmodule Pleroma.Gun.Conn do
opts = opts =
opts opts
|> Enum.into(%{}) |> Enum.into(%{})
|> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) |> Map.put_new(:connect_timeout, pool_opts[:connect_timeout] || 5_000)
|> Map.put_new(:supervise, false) |> Map.put_new(:supervise, false)
|> maybe_add_tls_opts(uri) |> maybe_add_tls_opts(uri)
@ -50,7 +50,7 @@ defmodule Pleroma.Gun.Conn do
with open_opts <- Map.delete(opts, :tls_opts), with open_opts <- Map.delete(opts, :tls_opts),
{:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts), {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]), {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]),
stream <- Gun.connect(conn, connect_opts), stream <- Gun.connect(conn, connect_opts),
{:response, :fin, 200, _} <- Gun.await(conn, stream) do {:response, :fin, 200, _} <- Gun.await(conn, stream) do
{:ok, conn} {:ok, conn}
@ -88,7 +88,7 @@ defmodule Pleroma.Gun.Conn do
|> Map.put(:socks_opts, socks_opts) |> Map.put(:socks_opts, socks_opts)
with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts), with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do
{:ok, conn} {:ok, conn}
else else
error -> error ->
@ -106,7 +106,7 @@ defmodule Pleroma.Gun.Conn do
host = Pleroma.HTTP.AdapterHelper.parse_host(host) host = Pleroma.HTTP.AdapterHelper.parse_host(host)
with {:ok, conn} <- Gun.open(host, port, opts), with {:ok, conn} <- Gun.open(host, port, opts),
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do
{:ok, conn} {:ok, conn}
else else
error -> error ->

View file

@ -6,7 +6,7 @@ defmodule Pleroma.HTTP.AdapterHelper do
@moduledoc """ @moduledoc """
Configure Tesla.Client with default and customized adapter options. Configure Tesla.Client with default and customized adapter options.
""" """
@defaults [pool: :federation] @defaults [pool: :federation, connect_timeout: 5_000, recv_timeout: 5_000]
@type proxy_type() :: :socks4 | :socks5 @type proxy_type() :: :socks4 | :socks5
@type host() :: charlist() | :inet.ip_address() @type host() :: charlist() | :inet.ip_address()

View file

@ -11,12 +11,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
require Logger require Logger
@defaults [ @defaults [
connect_timeout: 5_000,
domain_lookup_timeout: 5_000,
tls_handshake_timeout: 5_000,
retry: 1, retry: 1,
retry_timeout: 1000, retry_timeout: 1_000
await_up_timeout: 5_000
] ]
@type pool() :: :federation | :upload | :media | :default @type pool() :: :federation | :upload | :media | :default
@ -45,15 +41,17 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
end end
defp put_timeout(opts) do defp put_timeout(opts) do
{recv_timeout, opts} = Keyword.pop(opts, :recv_timeout, pool_timeout(opts[:pool]))
# this is the timeout to receive a message from Gun # this is the timeout to receive a message from Gun
Keyword.put_new(opts, :timeout, pool_timeout(opts[:pool])) # `:timeout` key is used in Tesla
Keyword.put(opts, :timeout, recv_timeout)
end end
@spec pool_timeout(pool()) :: non_neg_integer() @spec pool_timeout(pool()) :: non_neg_integer()
def pool_timeout(pool) do def pool_timeout(pool) do
default = Config.get([:pools, :default, :timeout], 5_000) default = Config.get([:pools, :default, :recv_timeout], 5_000)
Config.get([:pools, pool, :timeout], default) Config.get([:pools, pool, :recv_timeout], default)
end end
@prefix Pleroma.Gun.ConnectionPool @prefix Pleroma.Gun.ConnectionPool

View file

@ -2,11 +2,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
@behaviour Pleroma.HTTP.AdapterHelper @behaviour Pleroma.HTTP.AdapterHelper
@defaults [ @defaults [
connect_timeout: 10_000,
recv_timeout: 20_000,
follow_redirect: true, follow_redirect: true,
force_redirect: true, force_redirect: true
pool: :federation
] ]
@spec options(keyword(), URI.t()) :: keyword() @spec options(keyword(), URI.t()) :: keyword()
@ -19,6 +16,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
|> Keyword.merge(config_opts) |> Keyword.merge(config_opts)
|> Keyword.merge(connection_opts) |> Keyword.merge(connection_opts)
|> add_scheme_opts(uri) |> add_scheme_opts(uri)
|> maybe_add_with_body()
|> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy) |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
end end
@ -27,4 +25,12 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
end end
defp add_scheme_opts(opts, _), do: opts defp add_scheme_opts(opts, _), do: opts
defp maybe_add_with_body(opts) do
if opts[:max_body] do
Keyword.put(opts, :with_body, true)
else
opts
end
end
end end

View file

@ -13,22 +13,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
require Logger require Logger
@options [ @options [
pool: :media pool: :media,
recv_timeout: 10_000
] ]
def perform(:prefetch, url) do def perform(:prefetch, url) do
Logger.debug("Prefetching #{inspect(url)}") Logger.debug("Prefetching #{inspect(url)}")
opts =
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
Keyword.put(@options, :recv_timeout, 10_000)
else
@options
end
url url
|> MediaProxy.url() |> MediaProxy.url()
|> HTTP.get([], adapter: opts) |> HTTP.get([], adapter: @options)
end end
def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do

View file

@ -5,7 +5,8 @@
defmodule Pleroma.Web.RelMe do defmodule Pleroma.Web.RelMe do
@options [ @options [
pool: :media, pool: :media,
max_body: 2_000_000 max_body: 2_000_000,
recv_timeout: 2_000
] ]
if Pleroma.Config.get(:env) == :test do if Pleroma.Config.get(:env) == :test do
@ -23,18 +24,8 @@ defmodule Pleroma.Web.RelMe do
def parse(_), do: {:error, "No URL provided"} def parse(_), do: {:error, "No URL provided"}
defp parse_url(url) do defp parse_url(url) do
opts =
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
Keyword.merge(@options,
recv_timeout: 2_000,
with_body: true
)
else
@options
end
with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <- with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <-
Pleroma.HTTP.get(url, [], adapter: opts), Pleroma.HTTP.get(url, [], adapter: @options),
{:ok, html_tree} <- Floki.parse_document(html), {:ok, html_tree} <- Floki.parse_document(html),
data <- data <-
Floki.attribute(html_tree, "link[rel~=me]", "href") ++ Floki.attribute(html_tree, "link[rel~=me]", "href") ++

View file

@ -9,14 +9,15 @@ defmodule Pleroma.Web.RichMedia.Helpers do
alias Pleroma.Object alias Pleroma.Object
alias Pleroma.Web.RichMedia.Parser alias Pleroma.Web.RichMedia.Parser
@rich_media_options [ @options [
pool: :media, pool: :media,
max_body: 2_000_000 max_body: 2_000_000,
recv_timeout: 2_000
] ]
@spec validate_page_url(URI.t() | binary()) :: :ok | :error @spec validate_page_url(URI.t() | binary()) :: :ok | :error
defp validate_page_url(page_url) when is_binary(page_url) do defp validate_page_url(page_url) when is_binary(page_url) do
validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld]) validate_tld = Config.get([Pleroma.Formatter, :validate_tld])
page_url page_url
|> Linkify.Parser.url?(validate_tld: validate_tld) |> Linkify.Parser.url?(validate_tld: validate_tld)
@ -86,16 +87,6 @@ defmodule Pleroma.Web.RichMedia.Helpers do
def rich_media_get(url) do def rich_media_get(url) do
headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}] headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}]
options = Pleroma.HTTP.get(url, headers, adapter: @options)
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
Keyword.merge(@rich_media_options,
recv_timeout: 2_000,
with_body: true
)
else
@rich_media_options
end
Pleroma.HTTP.get(url, headers, adapter: options)
end end
end end