open conn in separate task

This commit is contained in:
Alexander Strizhakov 2020-02-24 19:56:27 +03:00
parent d44f9e3b6c
commit 8efae966b1
No known key found for this signature in database
GPG key ID: 022896A53AEF1381
12 changed files with 384 additions and 304 deletions

View file

@ -79,7 +79,7 @@ defmodule Mix.Tasks.Pleroma.Benchmark do
start_pleroma() start_pleroma()
:ok = :ok =
Pleroma.Pool.Connections.open_conn( Pleroma.Gun.Conn.open(
"https://httpbin.org/stream-bytes/1500", "https://httpbin.org/stream-bytes/1500",
:gun_connections :gun_connections
) )

View file

@ -6,9 +6,10 @@ defmodule Pleroma.Gun.API do
@callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()}
@callback info(pid()) :: map() @callback info(pid()) :: map()
@callback close(pid()) :: :ok @callback close(pid()) :: :ok
@callback await_up(pid) :: {:ok, atom()} | {:error, atom()} @callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()}
@callback connect(pid(), map()) :: reference() @callback connect(pid(), map()) :: reference()
@callback await(pid(), reference()) :: {:response, :fin, 200, []} @callback await(pid(), reference()) :: {:response, :fin, 200, []}
@callback set_owner(pid(), pid()) :: :ok
def open(host, port, opts), do: api().open(host, port, opts) def open(host, port, opts), do: api().open(host, port, opts)
@ -16,11 +17,13 @@ defmodule Pleroma.Gun.API do
def close(pid), do: api().close(pid) def close(pid), do: api().close(pid)
def await_up(pid), do: api().await_up(pid) def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout)
def connect(pid, opts), do: api().connect(pid, opts) def connect(pid, opts), do: api().connect(pid, opts)
def await(pid, ref), do: api().await(pid, ref) def await(pid, ref), do: api().await(pid, ref)
def set_owner(pid, owner), do: api().set_owner(pid, owner)
defp api, do: Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun) defp api, do: Pleroma.Config.get([Pleroma.Gun.API], Pleroma.Gun)
end end

View file

@ -118,7 +118,10 @@ defmodule Pleroma.Gun.API.Mock do
end end
@impl API @impl API
def await_up(_pid), do: {:ok, :http} def await_up(_pid, _timeout), do: {:ok, :http}
@impl API
def set_owner(_pid, _owner), do: :ok
@impl API @impl API
def connect(pid, %{host: _, port: 80}) do def connect(pid, %{host: _, port: 80}) do

View file

@ -6,6 +6,11 @@ defmodule Pleroma.Gun.Conn do
@moduledoc """ @moduledoc """
Struct for gun connection data Struct for gun connection data
""" """
alias Pleroma.Gun.API
alias Pleroma.Pool.Connections
require Logger
@type gun_state :: :up | :down @type gun_state :: :up | :down
@type conn_state :: :active | :idle @type conn_state :: :active | :idle
@ -26,4 +31,145 @@ defmodule Pleroma.Gun.Conn do
last_reference: 0, last_reference: 0,
crf: 1, crf: 1,
retries: 0 retries: 0
@spec open(String.t() | URI.t(), atom(), keyword()) :: :ok | nil
def open(url, name, opts \\ [])
def open(url, name, opts) when is_binary(url), do: open(URI.parse(url), name, opts)
def open(%URI{} = uri, name, opts) do
pool_opts = Pleroma.Config.get([:connections_pool], [])
opts =
opts
|> Enum.into(%{})
|> Map.put_new(:retry, pool_opts[:retry] || 0)
|> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 100)
|> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000)
key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
Logger.debug("opening new connection #{Connections.compose_uri_log(uri)}")
conn_pid =
if Connections.count(name) < opts[:max_connection] do
do_open(uri, opts)
else
try_do_open(name, uri, opts)
end
if is_pid(conn_pid) do
conn = %Pleroma.Gun.Conn{
conn: conn_pid,
gun_state: :up,
conn_state: :active,
last_reference: :os.system_time(:second)
}
:ok = API.set_owner(conn_pid, Process.whereis(name))
Connections.add_conn(name, key, conn)
end
end
defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do
connect_opts =
uri
|> destination_opts()
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
with open_opts <- Map.delete(opts, :tls_opts),
{:ok, conn} <- API.open(proxy_host, proxy_port, open_opts),
{:ok, _} <- API.await_up(conn, opts[:await_up_timeout]),
stream <- API.connect(conn, connect_opts),
{:response, :fin, 200, _} <- API.await(conn, stream) do
conn
else
error ->
Logger.warn(
"Received error on opening connection with http proxy #{
Connections.compose_uri_log(uri)
} #{inspect(error)}"
)
nil
end
end
defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do
version =
proxy_type
|> to_string()
|> String.last()
|> case do
"4" -> 4
_ -> 5
end
socks_opts =
uri
|> destination_opts()
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
|> Map.put(:version, version)
opts =
opts
|> Map.put(:protocols, [:socks])
|> Map.put(:socks_opts, socks_opts)
with {:ok, conn} <- API.open(proxy_host, proxy_port, opts),
{:ok, _} <- API.await_up(conn, opts[:await_up_timeout]) do
conn
else
error ->
Logger.warn(
"Received error on opening connection with socks proxy #{
Connections.compose_uri_log(uri)
} #{inspect(error)}"
)
nil
end
end
defp do_open(%URI{host: host, port: port} = uri, opts) do
{_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host)
with {:ok, conn} <- API.open(host, port, opts),
{:ok, _} <- API.await_up(conn, opts[:await_up_timeout]) do
conn
else
error ->
Logger.warn(
"Received error on opening connection #{Connections.compose_uri_log(uri)} #{
inspect(error)
}"
)
nil
end
end
defp destination_opts(%URI{host: host, port: port}) do
{_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host)
%{host: host, port: port}
end
defp add_http2_opts(opts, "https", tls_opts) do
Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts})
end
defp add_http2_opts(opts, _, _), do: opts
defp try_do_open(name, uri, opts) do
Logger.debug("try to open conn #{Connections.compose_uri_log(uri)}")
with [{close_key, least_used} | _conns] <-
Connections.get_unused_conns(name),
:ok <- Pleroma.Gun.API.close(least_used.conn) do
Connections.remove_conn(name, close_key)
do_open(uri, opts)
else
[] -> nil
end
end
end end

View file

@ -32,7 +32,7 @@ defmodule Pleroma.Gun do
defdelegate close(pid), to: :gun defdelegate close(pid), to: :gun
@impl API @impl API
defdelegate await_up(pid), to: :gun defdelegate await_up(pid, timeout \\ 5_000), to: :gun
@impl API @impl API
defdelegate connect(pid, opts), to: :gun defdelegate connect(pid, opts), to: :gun
@ -42,4 +42,7 @@ defmodule Pleroma.Gun do
@spec flush(pid() | reference()) :: :ok @spec flush(pid() | reference()) :: :ok
defdelegate flush(pid), to: :gun defdelegate flush(pid), to: :gun
@impl API
defdelegate set_owner(pid, owner), to: :gun
end end

View file

@ -12,7 +12,7 @@ defmodule Pleroma.HTTP.Adapter.Gun do
alias Pleroma.Pool.Connections alias Pleroma.Pool.Connections
@defaults [ @defaults [
connect_timeout: 20_000, connect_timeout: 5_000,
domain_lookup_timeout: 5_000, domain_lookup_timeout: 5_000,
tls_handshake_timeout: 5_000, tls_handshake_timeout: 5_000,
retry: 0, retry: 0,
@ -94,13 +94,11 @@ defmodule Pleroma.HTTP.Adapter.Gun do
"Gun connections pool checkin was not successful. Trying to open conn for next request." "Gun connections pool checkin was not successful. Trying to open conn for next request."
) )
:ok = Connections.open_conn(uri, :gun_connections, opts) Task.start(fn -> Pleroma.Gun.Conn.open(uri, :gun_connections, opts) end)
opts opts
conn when is_pid(conn) -> conn when is_pid(conn) ->
Logger.debug( Logger.debug("received conn #{inspect(conn)} #{Connections.compose_uri_log(uri)}")
"received conn #{inspect(conn)} #{uri.scheme}://#{Connections.compose_uri(uri)}"
)
opts opts
|> Keyword.put(:conn, conn) |> Keyword.put(:conn, conn)
@ -109,13 +107,14 @@ defmodule Pleroma.HTTP.Adapter.Gun do
rescue rescue
error -> error ->
Logger.warn( Logger.warn(
"Gun connections pool checkin caused error #{uri.scheme}://#{ "Gun connections pool checkin caused error #{Connections.compose_uri_log(uri)} #{
Connections.compose_uri(uri) inspect(error)
} #{inspect(error)}" }"
) )
opts opts
catch catch
# TODO: here must be no timeouts
:exit, {:timeout, {_, operation, [_, {method, _}, _]}} -> :exit, {:timeout, {_, operation, [_, {method, _}, _]}} ->
{:message_queue_len, messages_len} = {:message_queue_len, messages_len} =
:gun_connections :gun_connections
@ -124,15 +123,15 @@ defmodule Pleroma.HTTP.Adapter.Gun do
Logger.warn( Logger.warn(
"Gun connections pool checkin with timeout error for #{operation} #{method} #{ "Gun connections pool checkin with timeout error for #{operation} #{method} #{
uri.scheme Connections.compose_uri_log(uri)
}://#{Connections.compose_uri(uri)}. Messages length: #{messages_len}" }. Messages length: #{messages_len}"
) )
opts opts
:exit, error -> :exit, error ->
Logger.warn( Logger.warn(
"Gun pool checkin exited with error #{uri.scheme}://#{Connections.compose_uri(uri)} #{ "Gun pool checkin exited with error #{Connections.compose_uri_log(uri)} #{
inspect(error) inspect(error)
}" }"
) )

View file

@ -20,7 +20,6 @@ defmodule Pleroma.Pool.Connections do
defstruct conns: %{}, opts: [] defstruct conns: %{}, opts: []
alias Pleroma.Gun.API alias Pleroma.Gun.API
alias Pleroma.Gun.Conn
@spec start_link({atom(), keyword()}) :: {:ok, pid()} @spec start_link({atom(), keyword()}) :: {:ok, pid()}
def start_link({name, opts}) do def start_link({name, opts}) do
@ -44,23 +43,6 @@ defmodule Pleroma.Pool.Connections do
) )
end end
@spec open_conn(String.t() | URI.t(), atom(), keyword()) :: :ok
def open_conn(url, name, opts \\ [])
def open_conn(url, name, opts) when is_binary(url), do: open_conn(URI.parse(url), name, opts)
def open_conn(%URI{} = uri, name, opts) do
pool_opts = Config.get([:connections_pool], [])
opts =
opts
|> Enum.into(%{})
|> Map.put_new(:retry, pool_opts[:retry] || 0)
|> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 100)
|> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000)
GenServer.cast(name, {:open_conn, %{opts: opts, uri: uri}})
end
@spec alive?(atom()) :: boolean() @spec alive?(atom()) :: boolean()
def alive?(name) do def alive?(name) do
pid = Process.whereis(name) pid = Process.whereis(name)
@ -72,23 +54,37 @@ defmodule Pleroma.Pool.Connections do
GenServer.call(name, :state) GenServer.call(name, :state)
end end
@spec count(atom()) :: pos_integer()
def count(name) do
GenServer.call(name, :count)
end
@spec get_unused_conns(atom()) :: [{domain(), conn()}]
def get_unused_conns(name) do
GenServer.call(name, :unused_conns)
end
@spec checkout(pid(), pid(), atom()) :: :ok @spec checkout(pid(), pid(), atom()) :: :ok
def checkout(conn, pid, name) do def checkout(conn, pid, name) do
GenServer.cast(name, {:checkout, conn, pid}) GenServer.cast(name, {:checkout, conn, pid})
end end
@spec add_conn(atom(), String.t(), Pleroma.Gun.Conn.t()) :: :ok
def add_conn(name, key, conn) do
GenServer.cast(name, {:add_conn, key, conn})
end
@spec remove_conn(atom(), String.t()) :: :ok
def remove_conn(name, key) do
GenServer.cast(name, {:remove_conn, key})
end
@impl true @impl true
def handle_cast({:open_conn, %{opts: opts, uri: uri}}, state) do def handle_cast({:add_conn, key, conn}, state) do
Logger.debug("opening new #{compose_uri(uri)}") state = put_in(state.conns[key], conn)
max_connections = state.opts[:max_connections]
key = compose_key(uri) Process.monitor(conn.conn)
{:noreply, state}
if Enum.count(state.conns) < max_connections do
open_conn(key, uri, state, opts)
else
try_to_open_conn(key, uri, state, opts)
end
end end
@impl true @impl true
@ -120,14 +116,20 @@ defmodule Pleroma.Pool.Connections do
{:noreply, state} {:noreply, state}
end end
@impl true
def handle_cast({:remove_conn, key}, state) do
state = put_in(state.conns, Map.delete(state.conns, key))
{:noreply, state}
end
@impl true @impl true
def handle_call({:checkin, uri}, from, state) do def handle_call({:checkin, uri}, from, state) do
Logger.debug("checkin #{compose_uri(uri)}") key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
key = compose_key(uri) Logger.debug("checkin #{key}")
case state.conns[key] do case state.conns[key] do
%{conn: conn, gun_state: gun_state} = current_conn when gun_state == :up -> %{conn: conn, gun_state: gun_state} = current_conn when gun_state == :up ->
Logger.debug("reusing conn #{compose_uri(uri)}") Logger.debug("reusing conn #{key}")
with time <- :os.system_time(:second), with time <- :os.system_time(:second),
last_reference <- time - current_conn.last_reference, last_reference <- time - current_conn.last_reference,
@ -154,12 +156,31 @@ defmodule Pleroma.Pool.Connections do
@impl true @impl true
def handle_call(:state, _from, state), do: {:reply, state, state} def handle_call(:state, _from, state), do: {:reply, state, state}
@impl true
def handle_call(:count, _from, state) do
{:reply, Enum.count(state.conns), state}
end
@impl true
def handle_call(:unused_conns, _from, state) do
unused_conns =
state.conns
|> Enum.filter(fn {_k, v} ->
v.conn_state == :idle and v.used_by == []
end)
|> Enum.sort(fn {_x_k, x}, {_y_k, y} ->
x.crf <= y.crf and x.last_reference <= y.last_reference
end)
{:reply, unused_conns, state}
end
@impl true @impl true
def handle_info({:gun_up, conn_pid, _protocol}, state) do def handle_info({:gun_up, conn_pid, _protocol}, state) do
state = state =
with true <- Process.alive?(conn_pid), with conn_key when is_binary(conn_key) <- compose_key_gun_info(conn_pid),
conn_key when is_binary(conn_key) <- compose_key_gun_info(conn_pid),
{key, conn} <- find_conn(state.conns, conn_pid, conn_key), {key, conn} <- find_conn(state.conns, conn_pid, conn_key),
{true, key} <- {Process.alive?(conn_pid), key},
time <- :os.system_time(:second), time <- :os.system_time(:second),
last_reference <- time - conn.last_reference, last_reference <- time - conn.last_reference,
current_crf <- crf(last_reference, 100, conn.crf) do current_crf <- crf(last_reference, 100, conn.crf) do
@ -176,14 +197,16 @@ defmodule Pleroma.Pool.Connections do
Logger.debug(":gun.info caused error") Logger.debug(":gun.info caused error")
state state
false -> {false, key} ->
Logger.debug(":gun_up message for closed conn #{inspect(conn_pid)}") Logger.debug(":gun_up message for closed conn #{inspect(conn_pid)}")
state
put_in(
state.conns,
Map.delete(state.conns, key)
)
nil -> nil ->
Logger.debug( Logger.debug(":gun_up message for conn which is not found in state")
":gun_up message for alive conn #{inspect(conn_pid)}, but deleted from state"
)
:ok = API.close(conn_pid) :ok = API.close(conn_pid)
@ -198,8 +221,8 @@ defmodule Pleroma.Pool.Connections do
retries = Config.get([:connections_pool, :retry], 0) retries = Config.get([:connections_pool, :retry], 0)
# we can't get info on this pid, because pid is dead # we can't get info on this pid, because pid is dead
state = state =
with true <- Process.alive?(conn_pid), with {key, conn} <- find_conn(state.conns, conn_pid),
{key, conn} <- find_conn(state.conns, conn_pid) do {true, key} <- {Process.alive?(conn_pid), key} do
if conn.retries == retries do if conn.retries == retries do
Logger.debug("closing conn if retries is eq #{inspect(conn_pid)}") Logger.debug("closing conn if retries is eq #{inspect(conn_pid)}")
:ok = API.close(conn.conn) :ok = API.close(conn.conn)
@ -216,15 +239,17 @@ defmodule Pleroma.Pool.Connections do
}) })
end end
else else
false -> {false, key} ->
# gun can send gun_down for closed conn, maybe connection is not closed yet # gun can send gun_down for closed conn, maybe connection is not closed yet
Logger.debug(":gun_down message for closed conn #{inspect(conn_pid)}") Logger.debug(":gun_down message for closed conn #{inspect(conn_pid)}")
state
put_in(
state.conns,
Map.delete(state.conns, key)
)
nil -> nil ->
Logger.debug( Logger.debug(":gun_down message for conn which is not found in state")
":gun_down message for alive conn #{inspect(conn_pid)}, but deleted from state"
)
:ok = API.close(conn_pid) :ok = API.close(conn_pid)
@ -234,7 +259,29 @@ defmodule Pleroma.Pool.Connections do
{:noreply, state} {:noreply, state}
end end
defp compose_key(%URI{scheme: scheme, host: host, port: port}), do: "#{scheme}:#{host}:#{port}" @impl true
def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do
Logger.debug("received DOWM message for #{inspect(conn_pid)} reason -> #{inspect(reason)}")
state =
with {key, conn} <- find_conn(state.conns, conn_pid) do
Enum.each(conn.used_by, fn {pid, _ref} ->
Process.exit(pid, reason)
end)
put_in(
state.conns,
Map.delete(state.conns, key)
)
else
nil ->
Logger.debug(":DOWN message for conn which is not found in state")
state
end
{:noreply, state}
end
defp compose_key_gun_info(pid) do defp compose_key_gun_info(pid) do
try do try do
@ -265,153 +312,11 @@ defmodule Pleroma.Pool.Connections do
end) end)
end end
defp open_conn(key, uri, state, %{proxy: {proxy_host, proxy_port}} = opts) do
connect_opts =
uri
|> destination_opts()
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
with open_opts <- Map.delete(opts, :tls_opts),
{:ok, conn} <- API.open(proxy_host, proxy_port, open_opts),
{:ok, _} <- API.await_up(conn),
stream <- API.connect(conn, connect_opts),
{:response, :fin, 200, _} <- API.await(conn, stream),
state <-
put_in(state.conns[key], %Conn{
conn: conn,
gun_state: :up,
conn_state: :active,
last_reference: :os.system_time(:second)
}) do
{:noreply, state}
else
error ->
Logger.warn(
"Received error on opening connection with http proxy #{uri.scheme}://#{
compose_uri(uri)
}: #{inspect(error)}"
)
{:noreply, state}
end
end
defp open_conn(key, uri, state, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do
version =
proxy_type
|> to_string()
|> String.last()
|> case do
"4" -> 4
_ -> 5
end
socks_opts =
uri
|> destination_opts()
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
|> Map.put(:version, version)
opts =
opts
|> Map.put(:protocols, [:socks])
|> Map.put(:socks_opts, socks_opts)
with {:ok, conn} <- API.open(proxy_host, proxy_port, opts),
{:ok, _} <- API.await_up(conn),
state <-
put_in(state.conns[key], %Conn{
conn: conn,
gun_state: :up,
conn_state: :active,
last_reference: :os.system_time(:second)
}) do
{:noreply, state}
else
error ->
Logger.warn(
"Received error on opening connection with socks proxy #{uri.scheme}://#{
compose_uri(uri)
}: #{inspect(error)}"
)
{:noreply, state}
end
end
defp open_conn(key, %URI{host: host, port: port} = uri, state, opts) do
Logger.debug("opening conn #{compose_uri(uri)}")
{_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host)
with {:ok, conn} <- API.open(host, port, opts),
{:ok, _} <- API.await_up(conn),
state <-
put_in(state.conns[key], %Conn{
conn: conn,
gun_state: :up,
conn_state: :active,
last_reference: :os.system_time(:second)
}) do
Logger.debug("new conn opened #{compose_uri(uri)}")
Logger.debug("replying to the call #{compose_uri(uri)}")
{:noreply, state}
else
error ->
Logger.warn(
"Received error on opening connection #{uri.scheme}://#{compose_uri(uri)}: #{
inspect(error)
}"
)
{:noreply, state}
end
end
defp destination_opts(%URI{host: host, port: port}) do
{_type, host} = Pleroma.HTTP.Adapter.domain_or_ip(host)
%{host: host, port: port}
end
defp add_http2_opts(opts, "https", tls_opts) do
Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts})
end
defp add_http2_opts(opts, _, _), do: opts
@spec get_unused_conns(map()) :: [{domain(), conn()}]
def get_unused_conns(conns) do
conns
|> Enum.filter(fn {_k, v} ->
v.conn_state == :idle and v.used_by == []
end)
|> Enum.sort(fn {_x_k, x}, {_y_k, y} ->
x.crf <= y.crf and x.last_reference <= y.last_reference
end)
end
defp try_to_open_conn(key, uri, state, opts) do
Logger.debug("try to open conn #{compose_uri(uri)}")
with [{close_key, least_used} | _conns] <- get_unused_conns(state.conns),
:ok <- API.close(least_used.conn),
state <-
put_in(
state.conns,
Map.delete(state.conns, close_key)
) do
Logger.debug(
"least used conn found and closed #{inspect(least_used.conn)} #{compose_uri(uri)}"
)
open_conn(key, uri, state, opts)
else
[] -> {:noreply, state}
end
end
def crf(current, steps, crf) do def crf(current, steps, crf) do
1 + :math.pow(0.5, current / steps) * crf 1 + :math.pow(0.5, current / steps) * crf
end end
def compose_uri(%URI{} = uri), do: "#{uri.host}#{uri.path}" def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do
"#{scheme}://#{host}#{path}"
end
end end

View file

@ -44,7 +44,7 @@ defmodule Restarter.Pleroma do
end end
def handle_cast({:restart, :test, _}, state) do def handle_cast({:restart, :test, _}, state) do
Logger.warn("pleroma restarted") Logger.warn("pleroma manually restarted")
{:noreply, Map.put(state, :need_reboot?, false)} {:noreply, Map.put(state, :need_reboot?, false)}
end end
@ -57,7 +57,7 @@ defmodule Restarter.Pleroma do
def handle_cast({:after_boot, _}, %{after_boot: true} = state), do: {:noreply, state} def handle_cast({:after_boot, _}, %{after_boot: true} = state), do: {:noreply, state}
def handle_cast({:after_boot, :test}, state) do def handle_cast({:after_boot, :test}, state) do
Logger.warn("pleroma restarted") Logger.warn("pleroma restarted after boot")
{:noreply, Map.put(state, :after_boot, true)} {:noreply, Map.put(state, :after_boot, true)}
end end

View file

@ -19,6 +19,12 @@ defmodule Pleroma.GunTest do
assert json = receive_response(conn, ref) assert json = receive_response(conn, ref)
assert %{"args" => %{"a" => "b", "c" => "d"}} = Jason.decode!(json) assert %{"args" => %{"a" => "b", "c" => "d"}} = Jason.decode!(json)
{:ok, pid} = Task.start(fn -> Process.sleep(50) end)
:ok = :gun.set_owner(conn, pid)
assert :gun.info(conn).owner == pid
end end
defp receive_response(conn, ref, acc \\ "") do defp receive_response(conn, ref, acc \\ "") do

View file

@ -7,6 +7,7 @@ defmodule Pleroma.HTTP.Adapter.GunTest do
use Pleroma.Tests.Helpers use Pleroma.Tests.Helpers
import ExUnit.CaptureLog import ExUnit.CaptureLog
alias Pleroma.Config alias Pleroma.Config
alias Pleroma.Gun.Conn
alias Pleroma.HTTP.Adapter.Gun alias Pleroma.HTTP.Adapter.Gun
alias Pleroma.Pool.Connections alias Pleroma.Pool.Connections
@ -72,7 +73,7 @@ defmodule Pleroma.HTTP.Adapter.GunTest do
test "receive conn by default" do test "receive conn by default" do
uri = URI.parse("http://another-domain.com") uri = URI.parse("http://another-domain.com")
:ok = Connections.open_conn(uri, :gun_connections) :ok = Conn.open(uri, :gun_connections)
received_opts = Gun.options(uri) received_opts = Gun.options(uri)
assert received_opts[:close_conn] == false assert received_opts[:close_conn] == false
@ -81,7 +82,7 @@ defmodule Pleroma.HTTP.Adapter.GunTest do
test "don't receive conn if receive_conn is false" do test "don't receive conn if receive_conn is false" do
uri = URI.parse("http://another-domain2.com") uri = URI.parse("http://another-domain2.com")
:ok = Connections.open_conn(uri, :gun_connections) :ok = Conn.open(uri, :gun_connections)
opts = [receive_conn: false] opts = [receive_conn: false]
received_opts = Gun.options(opts, uri) received_opts = Gun.options(opts, uri)
@ -118,7 +119,7 @@ defmodule Pleroma.HTTP.Adapter.GunTest do
test "default ssl adapter opts with connection" do test "default ssl adapter opts with connection" do
uri = URI.parse("https://some-domain.com") uri = URI.parse("https://some-domain.com")
:ok = Connections.open_conn(uri, :gun_connections) :ok = Conn.open(uri, :gun_connections)
opts = Gun.options(uri) opts = Gun.options(uri)
@ -167,7 +168,7 @@ defmodule Pleroma.HTTP.Adapter.GunTest do
describe "after_request/1" do describe "after_request/1" do
test "body_as not chunks" do test "body_as not chunks" do
uri = URI.parse("http://some-domain.com") uri = URI.parse("http://some-domain.com")
:ok = Connections.open_conn(uri, :gun_connections) :ok = Conn.open(uri, :gun_connections)
opts = Gun.options(uri) opts = Gun.options(uri)
:ok = Gun.after_request(opts) :ok = Gun.after_request(opts)
conn = opts[:conn] conn = opts[:conn]
@ -185,7 +186,7 @@ defmodule Pleroma.HTTP.Adapter.GunTest do
test "body_as chunks" do test "body_as chunks" do
uri = URI.parse("http://some-domain.com") uri = URI.parse("http://some-domain.com")
:ok = Connections.open_conn(uri, :gun_connections) :ok = Conn.open(uri, :gun_connections)
opts = Gun.options([body_as: :chunks], uri) opts = Gun.options([body_as: :chunks], uri)
:ok = Gun.after_request(opts) :ok = Gun.after_request(opts)
conn = opts[:conn] conn = opts[:conn]
@ -205,7 +206,7 @@ defmodule Pleroma.HTTP.Adapter.GunTest do
test "with no connection" do test "with no connection" do
uri = URI.parse("http://uniq-domain.com") uri = URI.parse("http://uniq-domain.com")
:ok = Connections.open_conn(uri, :gun_connections) :ok = Conn.open(uri, :gun_connections)
opts = Gun.options([body_as: :chunks], uri) opts = Gun.options([body_as: :chunks], uri)
conn = opts[:conn] conn = opts[:conn]
@ -227,7 +228,7 @@ defmodule Pleroma.HTTP.Adapter.GunTest do
test "with ipv4" do test "with ipv4" do
uri = URI.parse("http://127.0.0.1") uri = URI.parse("http://127.0.0.1")
:ok = Connections.open_conn(uri, :gun_connections) :ok = Conn.open(uri, :gun_connections)
opts = Gun.options(uri) opts = Gun.options(uri)
send(:gun_connections, {:gun_up, opts[:conn], :http}) send(:gun_connections, {:gun_up, opts[:conn], :http})
:ok = Gun.after_request(opts) :ok = Gun.after_request(opts)
@ -246,7 +247,7 @@ defmodule Pleroma.HTTP.Adapter.GunTest do
test "with ipv6" do test "with ipv6" do
uri = URI.parse("http://[2a03:2880:f10c:83:face:b00c:0:25de]") uri = URI.parse("http://[2a03:2880:f10c:83:face:b00c:0:25de]")
:ok = Connections.open_conn(uri, :gun_connections) :ok = Conn.open(uri, :gun_connections)
opts = Gun.options(uri) opts = Gun.options(uri)
send(:gun_connections, {:gun_up, opts[:conn], :http}) send(:gun_connections, {:gun_up, opts[:conn], :http})
:ok = Gun.after_request(opts) :ok = Gun.after_request(opts)

View file

@ -124,7 +124,7 @@ defmodule Pleroma.HTTP.ConnectionTest do
uri = URI.parse("https://some-domain.com") uri = URI.parse("https://some-domain.com")
pid = Process.whereis(:federation) pid = Process.whereis(:federation)
:ok = Pleroma.Pool.Connections.open_conn(uri, :gun_connections, genserver_pid: pid) :ok = Pleroma.Gun.Conn.open(uri, :gun_connections, genserver_pid: pid)
opts = Connection.options(uri) opts = Connection.options(uri)

View file

@ -45,7 +45,7 @@ defmodule Pleroma.Pool.ConnectionsTest do
url = "http://some-domain.com" url = "http://some-domain.com"
key = "http:some-domain.com:80" key = "http:some-domain.com:80"
refute Connections.checkin(url, name) refute Connections.checkin(url, name)
:ok = Connections.open_conn(url, name) :ok = Conn.open(url, name)
conn = Connections.checkin(url, name) conn = Connections.checkin(url, name)
assert is_pid(conn) assert is_pid(conn)
@ -110,7 +110,7 @@ defmodule Pleroma.Pool.ConnectionsTest do
url = "http://ですsome-domain.com" url = "http://ですsome-domain.com"
refute Connections.checkin(url, name) refute Connections.checkin(url, name)
:ok = Connections.open_conn(url, name) :ok = Conn.open(url, name)
conn = Connections.checkin(url, name) conn = Connections.checkin(url, name)
assert is_pid(conn) assert is_pid(conn)
@ -139,7 +139,7 @@ defmodule Pleroma.Pool.ConnectionsTest do
refute Connections.checkin(url, name) refute Connections.checkin(url, name)
:ok = Connections.open_conn(url, name) :ok = Conn.open(url, name)
conn = Connections.checkin(url, name) conn = Connections.checkin(url, name)
assert is_pid(conn) assert is_pid(conn)
@ -182,7 +182,7 @@ defmodule Pleroma.Pool.ConnectionsTest do
refute Connections.checkin(url, name) refute Connections.checkin(url, name)
:ok = Connections.open_conn(url, name) :ok = Conn.open(url, name)
conn = Connections.checkin(url, name) conn = Connections.checkin(url, name)
assert is_pid(conn) assert is_pid(conn)
@ -209,7 +209,7 @@ defmodule Pleroma.Pool.ConnectionsTest do
test "up and down ipv4", %{name: name} do test "up and down ipv4", %{name: name} do
self = self() self = self()
url = "http://127.0.0.1" url = "http://127.0.0.1"
:ok = Connections.open_conn(url, name) :ok = Conn.open(url, name)
conn = Connections.checkin(url, name) conn = Connections.checkin(url, name)
send(name, {:gun_down, conn, nil, nil, nil}) send(name, {:gun_down, conn, nil, nil, nil})
send(name, {:gun_up, conn, nil}) send(name, {:gun_up, conn, nil})
@ -229,7 +229,7 @@ defmodule Pleroma.Pool.ConnectionsTest do
test "up and down ipv6", %{name: name} do test "up and down ipv6", %{name: name} do
self = self() self = self()
url = "http://[2a03:2880:f10c:83:face:b00c:0:25de]" url = "http://[2a03:2880:f10c:83:face:b00c:0:25de]"
:ok = Connections.open_conn(url, name) :ok = Conn.open(url, name)
conn = Connections.checkin(url, name) conn = Connections.checkin(url, name)
send(name, {:gun_down, conn, nil, nil, nil}) send(name, {:gun_down, conn, nil, nil, nil})
send(name, {:gun_up, conn, nil}) send(name, {:gun_up, conn, nil})
@ -253,13 +253,13 @@ defmodule Pleroma.Pool.ConnectionsTest do
https_key = "https:some-domain.com:443" https_key = "https:some-domain.com:443"
refute Connections.checkin(http_url, name) refute Connections.checkin(http_url, name)
:ok = Connections.open_conn(http_url, name) :ok = Conn.open(http_url, name)
conn = Connections.checkin(http_url, name) conn = Connections.checkin(http_url, name)
assert is_pid(conn) assert is_pid(conn)
assert Process.alive?(conn) assert Process.alive?(conn)
refute Connections.checkin(https_url, name) refute Connections.checkin(https_url, name)
:ok = Connections.open_conn(https_url, name) :ok = Conn.open(https_url, name)
https_conn = Connections.checkin(https_url, name) https_conn = Connections.checkin(https_url, name)
refute conn == https_conn refute conn == https_conn
@ -288,17 +288,17 @@ defmodule Pleroma.Pool.ConnectionsTest do
url = "http://gun-not-up.com" url = "http://gun-not-up.com"
assert capture_log(fn -> assert capture_log(fn ->
:ok = Connections.open_conn(url, name) refute Conn.open(url, name)
refute Connections.checkin(url, name) refute Connections.checkin(url, name)
end) =~ end) =~
"Received error on opening connection http://gun-not-up.com: {:error, :timeout}" "Received error on opening connection http://gun-not-up.com {:error, :timeout}"
end end
test "process gun_down message and then gun_up", %{name: name} do test "process gun_down message and then gun_up", %{name: name} do
self = self() self = self()
url = "http://gun-down-and-up.com" url = "http://gun-down-and-up.com"
key = "http:gun-down-and-up.com:80" key = "http:gun-down-and-up.com:80"
:ok = Connections.open_conn(url, name) :ok = Conn.open(url, name)
conn = Connections.checkin(url, name) conn = Connections.checkin(url, name)
assert is_pid(conn) assert is_pid(conn)
@ -347,7 +347,7 @@ defmodule Pleroma.Pool.ConnectionsTest do
test "async processes get same conn for same domain", %{name: name} do test "async processes get same conn for same domain", %{name: name} do
url = "http://some-domain.com" url = "http://some-domain.com"
:ok = Connections.open_conn(url, name) :ok = Conn.open(url, name)
tasks = tasks =
for _ <- 1..5 do for _ <- 1..5 do
@ -381,8 +381,8 @@ defmodule Pleroma.Pool.ConnectionsTest do
self = self() self = self()
http_url = "http://some-domain.com" http_url = "http://some-domain.com"
https_url = "https://some-domain.com" https_url = "https://some-domain.com"
:ok = Connections.open_conn(https_url, name) :ok = Conn.open(https_url, name)
:ok = Connections.open_conn(http_url, name) :ok = Conn.open(http_url, name)
conn1 = Connections.checkin(https_url, name) conn1 = Connections.checkin(https_url, name)
@ -413,7 +413,7 @@ defmodule Pleroma.Pool.ConnectionsTest do
:ok = Connections.checkout(conn1, self, name) :ok = Connections.checkout(conn1, self, name)
another_url = "http://another-domain.com" another_url = "http://another-domain.com"
:ok = Connections.open_conn(another_url, name) :ok = Conn.open(another_url, name)
conn = Connections.checkin(another_url, name) conn = Connections.checkin(another_url, name)
%Connections{ %Connections{
@ -437,9 +437,19 @@ defmodule Pleroma.Pool.ConnectionsTest do
Pleroma.Config.put(API, Pleroma.Gun) Pleroma.Config.put(API, Pleroma.Gun)
end end
test "opens connection and change owner", %{name: name} do
url = "https://httpbin.org"
:ok = Conn.open(url, name)
conn = Connections.checkin(url, name)
pid = Process.whereis(name)
assert :gun.info(conn).owner == pid
end
test "opens connection and reuse it on next request", %{name: name} do test "opens connection and reuse it on next request", %{name: name} do
url = "http://httpbin.org" url = "http://httpbin.org"
:ok = Connections.open_conn(url, name) :ok = Conn.open(url, name)
Process.sleep(250) Process.sleep(250)
conn = Connections.checkin(url, name) conn = Connections.checkin(url, name)
@ -462,7 +472,7 @@ defmodule Pleroma.Pool.ConnectionsTest do
test "opens ssl connection and reuse it on next request", %{name: name} do test "opens ssl connection and reuse it on next request", %{name: name} do
url = "https://httpbin.org" url = "https://httpbin.org"
:ok = Connections.open_conn(url, name) :ok = Conn.open(url, name)
Process.sleep(1_000) Process.sleep(1_000)
conn = Connections.checkin(url, name) conn = Connections.checkin(url, name)
@ -488,8 +498,8 @@ defmodule Pleroma.Pool.ConnectionsTest do
https1 = "https://www.google.com" https1 = "https://www.google.com"
https2 = "https://httpbin.org" https2 = "https://httpbin.org"
:ok = Connections.open_conn(https1, name) :ok = Conn.open(https1, name)
:ok = Connections.open_conn(https2, name) :ok = Conn.open(https2, name)
Process.sleep(1_500) Process.sleep(1_500)
conn = Connections.checkin(https1, name) conn = Connections.checkin(https1, name)
@ -513,7 +523,7 @@ defmodule Pleroma.Pool.ConnectionsTest do
:ok = Connections.checkout(conn, self, name) :ok = Connections.checkout(conn, self, name)
http = "http://httpbin.org" http = "http://httpbin.org"
Process.sleep(1_000) Process.sleep(1_000)
:ok = Connections.open_conn(http, name) :ok = Conn.open(http, name)
conn = Connections.checkin(http, name) conn = Connections.checkin(http, name)
%Connections{ %Connections{
@ -535,8 +545,8 @@ defmodule Pleroma.Pool.ConnectionsTest do
https1 = "https://www.google.com" https1 = "https://www.google.com"
https2 = "https://httpbin.org" https2 = "https://httpbin.org"
:ok = Connections.open_conn(https1, name) :ok = Conn.open(https1, name)
:ok = Connections.open_conn(https2, name) :ok = Conn.open(https2, name)
Process.sleep(1_500) Process.sleep(1_500)
Connections.checkin(https1, name) Connections.checkin(https1, name)
@ -563,7 +573,7 @@ defmodule Pleroma.Pool.ConnectionsTest do
:ok = Connections.checkout(conn, self, name) :ok = Connections.checkout(conn, self, name)
http = "http://httpbin.org" http = "http://httpbin.org"
:ok = Connections.open_conn(http, name) :ok = Conn.open(http, name)
Process.sleep(1_000) Process.sleep(1_000)
conn = Connections.checkin(http, name) conn = Connections.checkin(http, name)
@ -587,8 +597,8 @@ defmodule Pleroma.Pool.ConnectionsTest do
https1 = "https://www.google.com" https1 = "https://www.google.com"
https2 = "https://httpbin.org" https2 = "https://httpbin.org"
:ok = Connections.open_conn(https1, name) :ok = Conn.open(https1, name)
:ok = Connections.open_conn(https2, name) :ok = Conn.open(https2, name)
Process.sleep(1_000) Process.sleep(1_000)
Connections.checkin(https1, name) Connections.checkin(https1, name)
conn1 = Connections.checkin(https1, name) conn1 = Connections.checkin(https1, name)
@ -639,8 +649,8 @@ defmodule Pleroma.Pool.ConnectionsTest do
https1 = "https://www.google.com" https1 = "https://www.google.com"
https2 = "https://httpbin.org" https2 = "https://httpbin.org"
:ok = Connections.open_conn(https1, name) :ok = Conn.open(https1, name)
:ok = Connections.open_conn(https2, name) :ok = Conn.open(https2, name)
Process.sleep(1_500) Process.sleep(1_500)
Connections.checkin(https1, name) Connections.checkin(https1, name)
Connections.checkin(https2, name) Connections.checkin(https2, name)
@ -694,7 +704,7 @@ defmodule Pleroma.Pool.ConnectionsTest do
} = Connections.get_state(name) } = Connections.get_state(name)
http = "http://httpbin.org" http = "http://httpbin.org"
:ok = Connections.open_conn(http, name) :ok = Conn.open(http, name)
Process.sleep(1_000) Process.sleep(1_000)
conn = Connections.checkin(http, name) conn = Connections.checkin(http, name)
@ -725,7 +735,7 @@ defmodule Pleroma.Pool.ConnectionsTest do
test "as ip", %{name: name} do test "as ip", %{name: name} do
url = "http://proxy-string.com" url = "http://proxy-string.com"
key = "http:proxy-string.com:80" key = "http:proxy-string.com:80"
:ok = Connections.open_conn(url, name, proxy: {{127, 0, 0, 1}, 8123}) :ok = Conn.open(url, name, proxy: {{127, 0, 0, 1}, 8123})
conn = Connections.checkin(url, name) conn = Connections.checkin(url, name)
@ -745,7 +755,7 @@ defmodule Pleroma.Pool.ConnectionsTest do
test "as host", %{name: name} do test "as host", %{name: name} do
url = "http://proxy-tuple-atom.com" url = "http://proxy-tuple-atom.com"
:ok = Connections.open_conn(url, name, proxy: {'localhost', 9050}) :ok = Conn.open(url, name, proxy: {'localhost', 9050})
conn = Connections.checkin(url, name) conn = Connections.checkin(url, name)
%Connections{ %Connections{
@ -765,7 +775,7 @@ defmodule Pleroma.Pool.ConnectionsTest do
test "as ip and ssl", %{name: name} do test "as ip and ssl", %{name: name} do
url = "https://proxy-string.com" url = "https://proxy-string.com"
:ok = Connections.open_conn(url, name, proxy: {{127, 0, 0, 1}, 8123}) :ok = Conn.open(url, name, proxy: {{127, 0, 0, 1}, 8123})
conn = Connections.checkin(url, name) conn = Connections.checkin(url, name)
%Connections{ %Connections{
@ -784,7 +794,7 @@ defmodule Pleroma.Pool.ConnectionsTest do
test "as host and ssl", %{name: name} do test "as host and ssl", %{name: name} do
url = "https://proxy-tuple-atom.com" url = "https://proxy-tuple-atom.com"
:ok = Connections.open_conn(url, name, proxy: {'localhost', 9050}) :ok = Conn.open(url, name, proxy: {'localhost', 9050})
conn = Connections.checkin(url, name) conn = Connections.checkin(url, name)
%Connections{ %Connections{
@ -804,7 +814,7 @@ defmodule Pleroma.Pool.ConnectionsTest do
test "with socks type", %{name: name} do test "with socks type", %{name: name} do
url = "http://proxy-socks.com" url = "http://proxy-socks.com"
:ok = Connections.open_conn(url, name, proxy: {:socks5, 'localhost', 1234}) :ok = Conn.open(url, name, proxy: {:socks5, 'localhost', 1234})
conn = Connections.checkin(url, name) conn = Connections.checkin(url, name)
@ -825,7 +835,7 @@ defmodule Pleroma.Pool.ConnectionsTest do
test "with socks4 type and ssl", %{name: name} do test "with socks4 type and ssl", %{name: name} do
url = "https://proxy-socks.com" url = "https://proxy-socks.com"
:ok = Connections.open_conn(url, name, proxy: {:socks4, 'localhost', 1234}) :ok = Conn.open(url, name, proxy: {:socks4, 'localhost', 1234})
conn = Connections.checkin(url, name) conn = Connections.checkin(url, name)
@ -892,71 +902,75 @@ defmodule Pleroma.Pool.ConnectionsTest do
end end
describe "get_unused_conns/1" do describe "get_unused_conns/1" do
test "crf is equalent, sorting by reference" do test "crf is equalent, sorting by reference", %{name: name} do
conns = %{ Connections.add_conn(name, "1", %Conn{
"1" => %Conn{ conn_state: :idle,
conn_state: :idle, last_reference: now() - 1
last_reference: now() - 1 })
},
"2" => %Conn{
conn_state: :idle,
last_reference: now()
}
}
assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(conns) Connections.add_conn(name, "2", %Conn{
conn_state: :idle,
last_reference: now()
})
assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(name)
end end
test "reference is equalent, sorting by crf" do test "reference is equalent, sorting by crf", %{name: name} do
conns = %{ Connections.add_conn(name, "1", %Conn{
"1" => %Conn{ conn_state: :idle,
conn_state: :idle, crf: 1.999
crf: 1.999 })
},
"2" => %Conn{
conn_state: :idle,
crf: 2
}
}
assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(conns) Connections.add_conn(name, "2", %Conn{
conn_state: :idle,
crf: 2
})
assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(name)
end end
test "higher crf and lower reference" do test "higher crf and lower reference", %{name: name} do
conns = %{ Connections.add_conn(name, "1", %Conn{
"1" => %Conn{ conn_state: :idle,
conn_state: :idle, crf: 3,
crf: 3, last_reference: now() - 1
last_reference: now() - 1 })
},
"2" => %Conn{
conn_state: :idle,
crf: 2,
last_reference: now()
}
}
assert [{"2", _unused_conn} | _others] = Connections.get_unused_conns(conns) Connections.add_conn(name, "2", %Conn{
conn_state: :idle,
crf: 2,
last_reference: now()
})
assert [{"2", _unused_conn} | _others] = Connections.get_unused_conns(name)
end end
test "lower crf and lower reference" do test "lower crf and lower reference", %{name: name} do
conns = %{ Connections.add_conn(name, "1", %Conn{
"1" => %Conn{ conn_state: :idle,
conn_state: :idle, crf: 1.99,
crf: 1.99, last_reference: now() - 1
last_reference: now() - 1 })
},
"2" => %Conn{
conn_state: :idle,
crf: 2,
last_reference: now()
}
}
assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(conns) Connections.add_conn(name, "2", %Conn{
conn_state: :idle,
crf: 2,
last_reference: now()
})
assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(name)
end end
end end
test "count/1", %{name: name} do
assert Connections.count(name) == 0
Connections.add_conn(name, "1", %Conn{conn: self()})
assert Connections.count(name) == 1
Connections.remove_conn(name, "1")
assert Connections.count(name) == 0
end
defp now do defp now do
:os.system_time(:second) :os.system_time(:second)
end end