add finch outbound proxy support (#158)

Reviewed-on: AkkomaGang/akkoma#158

CHANGELOG.md

@@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
 
+### Removed
+- Non-finch HTTP adapters. `:tesla, :adapter` is now highly recommended to be set to the default.
+
+## 2022.08
+
 ### Added
 - extended runtime module support, see config cheatsheet
 - quote posting; quotes are limited to public posts

config/description.exs

@@ -2598,9 +2598,10 @@
       %{
         key: :proxy_url,
         label: "Proxy URL",
-        type: [:string, :tuple],
+        type: :string,
-        description: "Proxy URL",
+        description:
-        suggestions: ["localhost:9020", {:socks5, :localhost, 3090}]
+          "Proxy URL - of the format http://host:port. Advise setting in .exs instead of admin-fe due to this being set at boot-time.",
+        suggestions: ["http://localhost:3128"]
       },
       %{
         key: :user_agent,

docs/docs/configuration/cheatsheet.md

@@ -521,7 +521,7 @@ Available caches:
 
 ### :http
 
-* `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`)
+* `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`); for example `http://127.0.0.1:3192`. Does not support SOCKS5 proxy, only http(s).
 * `send_user_agent`: should we include a user agent with HTTP requests? (default: `true`)
 * `user_agent`: what user agent should we use? (default: `:default`), must be string or `:default`
 * `adapter`: array of adapter options

lib/pleroma/application.ex

@@ -248,9 +248,13 @@ def limiters_setup do
   end
 
   defp http_children do
+    proxy_url = Config.get([:http, :proxy_url])
+    proxy = Pleroma.HTTP.AdapterHelper.format_proxy(proxy_url)
+
     config =
       [:http, :adapter]
       |> Config.get([])
+      |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy_pool(proxy)
       |> Keyword.put(:name, MyFinch)
 
     [{Finch, config}]

lib/pleroma/http/adapter_helper.ex

@@ -6,7 +6,7 @@ defmodule Pleroma.HTTP.AdapterHelper do
   @moduledoc """
   Configure Tesla.Client with default and customized adapter options.
   """
-  @defaults [name: MyFinch, connect_timeout: 5_000, recv_timeout: 5_000]
+  @defaults [name: MyFinch, pool_timeout: 5_000, receive_timeout: 5_000]
 
   @type proxy_type() :: :socks4 | :socks5
   @type host() :: charlist() | :inet.ip_address()
@@ -25,15 +25,58 @@ def format_proxy(nil), do: nil
 
   def format_proxy(proxy_url) do
     case parse_proxy(proxy_url) do
-      {:ok, host, port} -> {host, port}
+      {:ok, host, port} -> {:http, host, port, []}
-      {:ok, type, host, port} -> {type, host, port}
+      {:ok, type, host, port} -> {type, host, port, []}
       _ -> nil
     end
   end
 
   @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword()
   def maybe_add_proxy(opts, nil), do: opts
-  def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy)
+
+  def maybe_add_proxy(opts, proxy) do
+    Keyword.put(opts, :proxy, proxy)
+  end
+
+  def maybe_add_proxy_pool(opts, nil), do: opts
+
+  def maybe_add_proxy_pool(opts, proxy) do
+    Logger.info("Using HTTP Proxy: #{inspect(proxy)}")
+
+    opts
+    |> maybe_add_pools()
+    |> maybe_add_default_pool()
+    |> maybe_add_conn_opts()
+    |> put_in([:pools, :default, :conn_opts, :proxy], proxy)
+  end
+
+  defp maybe_add_pools(opts) do
+    if Keyword.has_key?(opts, :pools) do
+      opts
+    else
+      Keyword.put(opts, :pools, %{})
+    end
+  end
+
+  defp maybe_add_default_pool(opts) do
+    pools = Keyword.get(opts, :pools)
+
+    if Map.has_key?(pools, :default) do
+      opts
+    else
+      put_in(opts, [:pools, :default], [])
+    end
+  end
+
+  defp maybe_add_conn_opts(opts) do
+    conn_opts = get_in(opts, [:pools, :default, :conn_opts])
+
+    unless is_nil(conn_opts) do
+      opts
+    else
+      put_in(opts, [:pools, :default, :conn_opts], [])
+    end
+  end
 
   @doc """
   Merge default connection & adapter options with received ones.
@@ -46,36 +89,31 @@ def options(%URI{} = uri, opts \\ []) do
     |> AdapterHelper.Default.options(uri)
   end
 
+  defp proxy_type("http"), do: {:ok, :http}
+  defp proxy_type("https"), do: {:ok, :https}
+  defp proxy_type(_), do: {:error, :unknown}
+
   @spec parse_proxy(String.t() | tuple() | nil) ::
           {:ok, host(), pos_integer()}
           | {:ok, proxy_type(), host(), pos_integer()}
           | {:error, atom()}
           | nil
-
   def parse_proxy(nil), do: nil
 
   def parse_proxy(proxy) when is_binary(proxy) do
-    with [host, port] <- String.split(proxy, ":"),
+    with %URI{} = uri <- URI.parse(proxy),
-         {port, ""} <- Integer.parse(port) do
+         {:ok, type} <- proxy_type(uri.scheme) do
-      {:ok, parse_host(host), port}
+      {:ok, type, uri.host, uri.port}
     else
-      {_, _} ->
+      e ->
-        Logger.warn("Parsing port failed #{inspect(proxy)}")
+        Logger.warn("Parsing proxy failed #{inspect(proxy)}, #{inspect(e)}")
-        {:error, :invalid_proxy_port}
-
-      :error ->
-        Logger.warn("Parsing port failed #{inspect(proxy)}")
-        {:error, :invalid_proxy_port}
-
-      _ ->
-        Logger.warn("Parsing proxy failed #{inspect(proxy)}")
         {:error, :invalid_proxy}
     end
   end
 
   def parse_proxy(proxy) when is_tuple(proxy) do
     with {type, host, port} <- proxy do
-      {:ok, type, parse_host(host), port}
+      {:ok, type, host, port}
     else
       _ ->
         Logger.warn("Parsing proxy failed #{inspect(proxy)}")

lib/pleroma/http/adapter_helper/default.ex

@@ -9,7 +9,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Default do
 
   @spec options(keyword(), URI.t()) :: keyword()
   def options(opts, _uri) do
-    proxy = Pleroma.Config.get([:http, :proxy_url], nil)
+    proxy = Pleroma.Config.get([:http, :proxy_url])
     AdapterHelper.maybe_add_proxy(opts, AdapterHelper.format_proxy(proxy))
   end
 

lib/pleroma/http/adapter_helper/gun.ex

@@ -1,82 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.HTTP.AdapterHelper.Gun do
-  @behaviour Pleroma.HTTP.AdapterHelper
-
-  alias Pleroma.Config
-  alias Pleroma.HTTP.AdapterHelper
-
-  require Logger
-
-  @defaults [
-    retry: 1,
-    retry_timeout: 1_000
-  ]
-
-  @type pool() :: :federation | :upload | :media | :default
-
-  @spec options(keyword(), URI.t()) :: keyword()
-  def options(incoming_opts \\ [], %URI{} = uri) do
-    proxy =
-      [:http, :proxy_url]
-      |> Config.get()
-      |> AdapterHelper.format_proxy()
-
-    config_opts = Config.get([:http, :adapter], [])
-
-    @defaults
-    |> Keyword.merge(config_opts)
-    |> add_scheme_opts(uri)
-    |> AdapterHelper.maybe_add_proxy(proxy)
-    |> Keyword.merge(incoming_opts)
-    |> put_timeout()
-  end
-
-  defp add_scheme_opts(opts, %{scheme: "http"}), do: opts
-
-  defp add_scheme_opts(opts, %{scheme: "https"}) do
-    Keyword.put(opts, :certificates_verification, true)
-  end
-
-  defp put_timeout(opts) do
-    {recv_timeout, opts} = Keyword.pop(opts, :recv_timeout, pool_timeout(opts[:pool]))
-    # this is the timeout to receive a message from Gun
-    # `:timeout` key is used in Tesla
-    Keyword.put(opts, :timeout, recv_timeout)
-  end
-
-  @spec pool_timeout(pool()) :: non_neg_integer()
-  def pool_timeout(pool) do
-    default = Config.get([:pools, :default, :recv_timeout], 5_000)
-
-    Config.get([:pools, pool, :recv_timeout], default)
-  end
-
-  def limiter_setup do
-    prefix = Pleroma.Gun.ConnectionPool
-    wait = Config.get([:connections_pool, :connection_acquisition_wait])
-    retries = Config.get([:connections_pool, :connection_acquisition_retries])
-
-    :pools
-    |> Config.get([])
-    |> Enum.each(fn {name, opts} ->
-      max_running = Keyword.get(opts, :size, 50)
-      max_waiting = Keyword.get(opts, :max_waiting, 10)
-
-      result =
-        ConcurrentLimiter.new(:"#{prefix}.#{name}", max_running, max_waiting,
-          wait: wait,
-          max_retries: retries
-        )
-
-      case result do
-        :ok -> :ok
-        {:error, :existing} -> :ok
-      end
-    end)
-
-    :ok
-  end
-end

lib/pleroma/http/adapter_helper/hackney.ex

@@ -1,40 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.HTTP.AdapterHelper.Hackney do
-  @behaviour Pleroma.HTTP.AdapterHelper
-
-  @defaults [
-    follow_redirect: true,
-    force_redirect: true
-  ]
-
-  @spec options(keyword(), URI.t()) :: keyword()
-  def options(connection_opts \\ [], %URI{} = uri) do
-    proxy = Pleroma.Config.get([:http, :proxy_url])
-
-    config_opts = Pleroma.Config.get([:http, :adapter], [])
-
-    @defaults
-    |> Keyword.merge(config_opts)
-    |> Keyword.merge(connection_opts)
-    |> add_scheme_opts(uri)
-    |> maybe_add_with_body()
-    |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
-  end
-
-  defp add_scheme_opts(opts, %URI{scheme: "https"}) do
-    Keyword.put(opts, :ssl_options, versions: [:"tlsv1.3", :"tlsv1.2", :"tlsv1.1", :tlsv1])
-  end
-
-  defp add_scheme_opts(opts, _), do: opts
-
-  defp maybe_add_with_body(opts) do
-    if opts[:max_body] do
-      Keyword.put(opts, :with_body, true)
-    else
-      opts
-    end
-  end
-end

test/pleroma/http/adapter_helper/gun_test.exs

@@ -1,77 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.HTTP.AdapterHelper.GunTest do
-  use ExUnit.Case
-  use Pleroma.Tests.Helpers
-
-  import Mox
-
-  alias Pleroma.HTTP.AdapterHelper.Gun
-
-  setup :verify_on_exit!
-
-  describe "options/1" do
-    setup do: clear_config([:http, :adapter], a: 1, b: 2)
-
-    test "https url with default port" do
-      uri = URI.parse("https://example.com")
-
-      opts = Gun.options([receive_conn: false], uri)
-      assert opts[:certificates_verification]
-    end
-
-    test "https ipv4 with default port" do
-      uri = URI.parse("https://127.0.0.1")
-
-      opts = Gun.options([receive_conn: false], uri)
-      assert opts[:certificates_verification]
-    end
-
-    test "https ipv6 with default port" do
-      uri = URI.parse("https://[2a03:2880:f10c:83:face:b00c:0:25de]")
-
-      opts = Gun.options([receive_conn: false], uri)
-      assert opts[:certificates_verification]
-    end
-
-    test "https url with non standart port" do
-      uri = URI.parse("https://example.com:115")
-
-      opts = Gun.options([receive_conn: false], uri)
-
-      assert opts[:certificates_verification]
-    end
-
-    test "merges with defaul http adapter config" do
-      defaults = Gun.options([receive_conn: false], URI.parse("https://example.com"))
-      assert Keyword.has_key?(defaults, :a)
-      assert Keyword.has_key?(defaults, :b)
-    end
-
-    test "parses string proxy host & port" do
-      clear_config([:http, :proxy_url], "localhost:8123")
-
-      uri = URI.parse("https://some-domain.com")
-      opts = Gun.options([receive_conn: false], uri)
-      assert opts[:proxy] == {'localhost', 8123}
-    end
-
-    test "parses tuple proxy scheme host and port" do
-      clear_config([:http, :proxy_url], {:socks, 'localhost', 1234})
-
-      uri = URI.parse("https://some-domain.com")
-      opts = Gun.options([receive_conn: false], uri)
-      assert opts[:proxy] == {:socks, 'localhost', 1234}
-    end
-
-    test "passed opts have more weight than defaults" do
-      clear_config([:http, :proxy_url], {:socks5, 'localhost', 1234})
-      uri = URI.parse("https://some-domain.com")
-      opts = Gun.options([receive_conn: false, proxy: {'example.com', 4321}], uri)
-
-      assert opts[:proxy] == {'example.com', 4321}
-    end
-  end
-end

test/pleroma/http/adapter_helper/hackney_test.exs

@@ -1,35 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.HTTP.AdapterHelper.HackneyTest do
-  use ExUnit.Case, async: true
-  use Pleroma.Tests.Helpers
-
-  alias Pleroma.HTTP.AdapterHelper.Hackney
-
-  setup_all do
-    uri = URI.parse("http://domain.com")
-    {:ok, uri: uri}
-  end
-
-  describe "options/2" do
-    setup do: clear_config([:http, :adapter], a: 1, b: 2)
-
-    test "add proxy and opts from config", %{uri: uri} do
-      opts = Hackney.options([proxy: "localhost:8123"], uri)
-
-      assert opts[:a] == 1
-      assert opts[:b] == 2
-      assert opts[:proxy] == "localhost:8123"
-    end
-
-    test "respect connection opts and no proxy", %{uri: uri} do
-      opts = Hackney.options([a: 2, b: 1], uri)
-
-      assert opts[:a] == 2
-      assert opts[:b] == 1
-      refute Keyword.has_key?(opts, :proxy)
-    end
-  end
-end

test/pleroma/http/adapter_helper_test.exs

@@ -13,16 +13,38 @@ test "with nil" do
     end
 
     test "with string" do
-      assert AdapterHelper.format_proxy("127.0.0.1:8123") == {{127, 0, 0, 1}, 8123}
+      assert AdapterHelper.format_proxy("http://127.0.0.1:8123") == {:http, "127.0.0.1", 8123, []}
     end
 
     test "localhost with port" do
-      assert AdapterHelper.format_proxy("localhost:8123") == {'localhost', 8123}
+      assert AdapterHelper.format_proxy("https://localhost:8123") ==
+               {:https, "localhost", 8123, []}
     end
 
     test "tuple" do
-      assert AdapterHelper.format_proxy({:socks4, :localhost, 9050}) ==
+      assert AdapterHelper.format_proxy({:http, "localhost", 9050}) ==
-               {:socks4, 'localhost', 9050}
+               {:http, "localhost", 9050, []}
+    end
+  end
+
+  describe "maybe_add_proxy_pool/1" do
+    test "should do nothing with nil" do
+      assert AdapterHelper.maybe_add_proxy_pool([], nil) == []
+    end
+
+    test "should create pools" do
+      assert AdapterHelper.maybe_add_proxy_pool([], "proxy") == [
+               pools: %{default: [conn_opts: [proxy: "proxy"]]}
+             ]
+    end
+
+    test "should not override conn_opts if set" do
+      assert AdapterHelper.maybe_add_proxy_pool(
+               [pools: %{default: [conn_opts: [already: "set"]]}],
+               "proxy"
+             ) == [
+               pools: %{default: [conn_opts: [proxy: "proxy", already: "set"]]}
+             ]
     end
   end
 end