Merge remote-tracking branch 'remotes/origin/develop' into output-of-relationships-in-statuses

# Conflicts:
#	CHANGELOG.md
This commit is contained in:
Ivan Tashkinov 2020-04-01 20:08:14 +03:00
commit 98b96e45ab
77 changed files with 3343 additions and 511 deletions

View file

@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
### Added ### Added
- NodeInfo: `pleroma:api/v1/notifications:include_types_filter` to the `features` list. - NodeInfo: `pleroma:api/v1/notifications:include_types_filter` to the `features` list.
- Configuration: `:restrict_unauthenticated` setting, restrict access for unauthenticated users to timelines (public and federate), user profiles and statuses. - Configuration: `:restrict_unauthenticated` setting, restrict access for unauthenticated users to timelines (public and federate), user profiles and statuses.
- New HTTP adapter [gun](https://github.com/ninenines/gun). Gun adapter requires minimum OTP version of 22.2 otherwise Pleroma wont start. For hackney OTP update is not required.
- Configuration: `:extensions/:output_relationships_in_statuses_by_default` option (if `false`, disables the output of account/pleroma/relationship for statuses and notifications by default, breaking the compatibility with older PleromaFE versions). - Configuration: `:extensions/:output_relationships_in_statuses_by_default` option (if `false`, disables the output of account/pleroma/relationship for statuses and notifications by default, breaking the compatibility with older PleromaFE versions).
<details> <details>
<summary>API Changes</summary> <summary>API Changes</summary>

View file

@ -58,20 +58,6 @@ config :pleroma, Pleroma.Captcha,
config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch" config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch"
config :pleroma, :hackney_pools,
federation: [
max_connections: 50,
timeout: 150_000
],
media: [
max_connections: 50,
timeout: 150_000
],
upload: [
max_connections: 25,
timeout: 300_000
]
# Upload configuration # Upload configuration
config :pleroma, Pleroma.Upload, config :pleroma, Pleroma.Upload,
uploader: Pleroma.Uploaders.Local, uploader: Pleroma.Uploaders.Local,
@ -184,21 +170,13 @@ config :mime, :types, %{
"application/ld+json" => ["activity+json"] "application/ld+json" => ["activity+json"]
} }
config :tesla, adapter: Tesla.Adapter.Hackney config :tesla, adapter: Tesla.Adapter.Gun
# Configures http settings, upstream proxy etc. # Configures http settings, upstream proxy etc.
config :pleroma, :http, config :pleroma, :http,
proxy_url: nil, proxy_url: nil,
send_user_agent: true, send_user_agent: true,
user_agent: :default, user_agent: :default,
adapter: [ adapter: []
ssl_options: [
# Workaround for remote server certificate chain issues
partial_chain: &:hackney_connect.partial_chain/1,
# We don't support TLS v1.3 yet
versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"]
]
]
config :pleroma, :instance, config :pleroma, :instance,
name: "Pleroma", name: "Pleroma",
@ -626,6 +604,49 @@ config :pleroma, Pleroma.Repo,
parameters: [gin_fuzzy_search_limit: "500"], parameters: [gin_fuzzy_search_limit: "500"],
prepare: :unnamed prepare: :unnamed
config :pleroma, :connections_pool,
checkin_timeout: 250,
max_connections: 250,
retry: 1,
retry_timeout: 1000,
await_up_timeout: 5_000
config :pleroma, :pools,
federation: [
size: 50,
max_overflow: 10,
timeout: 150_000
],
media: [
size: 50,
max_overflow: 10,
timeout: 150_000
],
upload: [
size: 25,
max_overflow: 5,
timeout: 300_000
],
default: [
size: 10,
max_overflow: 2,
timeout: 10_000
]
config :pleroma, :hackney_pools,
federation: [
max_connections: 50,
timeout: 150_000
],
media: [
max_connections: 50,
timeout: 150_000
],
upload: [
max_connections: 25,
timeout: 300_000
]
config :pleroma, :restrict_unauthenticated, config :pleroma, :restrict_unauthenticated,
timelines: %{local: false, federated: false}, timelines: %{local: false, federated: false},
profiles: %{local: false, remote: false}, profiles: %{local: false, remote: false},

View file

@ -2932,6 +2932,219 @@ config :pleroma, :config_description, [
} }
] ]
}, },
%{
group: :pleroma,
key: :connections_pool,
type: :group,
description: "Advanced settings for `gun` connections pool",
children: [
%{
key: :checkin_timeout,
type: :integer,
description: "Timeout to checkin connection from pool. Default: 250ms.",
suggestions: [250]
},
%{
key: :max_connections,
type: :integer,
description: "Maximum number of connections in the pool. Default: 250 connections.",
suggestions: [250]
},
%{
key: :retry,
type: :integer,
description:
"Number of retries, while `gun` will try to reconnect if connection goes down. Default: 1.",
suggestions: [1]
},
%{
key: :retry_timeout,
type: :integer,
description:
"Time between retries when `gun` will try to reconnect in milliseconds. Default: 1000ms.",
suggestions: [1000]
},
%{
key: :await_up_timeout,
type: :integer,
description: "Timeout while `gun` will wait until connection is up. Default: 5000ms.",
suggestions: [5000]
}
]
},
%{
group: :pleroma,
key: :pools,
type: :group,
description: "Advanced settings for `gun` workers pools",
children: [
%{
key: :federation,
type: :keyword,
description: "Settings for federation pool.",
children: [
%{
key: :size,
type: :integer,
description: "Number workers in the pool.",
suggestions: [50]
},
%{
key: :max_overflow,
type: :integer,
description: "Number of additional workers if pool is under load.",
suggestions: [10]
},
%{
key: :timeout,
type: :integer,
description: "Timeout while `gun` will wait for response.",
suggestions: [150_000]
}
]
},
%{
key: :media,
type: :keyword,
description: "Settings for media pool.",
children: [
%{
key: :size,
type: :integer,
description: "Number workers in the pool.",
suggestions: [50]
},
%{
key: :max_overflow,
type: :integer,
description: "Number of additional workers if pool is under load.",
suggestions: [10]
},
%{
key: :timeout,
type: :integer,
description: "Timeout while `gun` will wait for response.",
suggestions: [150_000]
}
]
},
%{
key: :upload,
type: :keyword,
description: "Settings for upload pool.",
children: [
%{
key: :size,
type: :integer,
description: "Number workers in the pool.",
suggestions: [25]
},
%{
key: :max_overflow,
type: :integer,
description: "Number of additional workers if pool is under load.",
suggestions: [5]
},
%{
key: :timeout,
type: :integer,
description: "Timeout while `gun` will wait for response.",
suggestions: [300_000]
}
]
},
%{
key: :default,
type: :keyword,
description: "Settings for default pool.",
children: [
%{
key: :size,
type: :integer,
description: "Number workers in the pool.",
suggestions: [10]
},
%{
key: :max_overflow,
type: :integer,
description: "Number of additional workers if pool is under load.",
suggestions: [2]
},
%{
key: :timeout,
type: :integer,
description: "Timeout while `gun` will wait for response.",
suggestions: [10_000]
}
]
}
]
},
%{
group: :pleroma,
key: :hackney_pools,
type: :group,
description: "Advanced settings for `hackney` connections pools",
children: [
%{
key: :federation,
type: :keyword,
description: "Settings for federation pool.",
children: [
%{
key: :max_connections,
type: :integer,
description: "Number workers in the pool.",
suggestions: [50]
},
%{
key: :timeout,
type: :integer,
description: "Timeout while `hackney` will wait for response.",
suggestions: [150_000]
}
]
},
%{
key: :media,
type: :keyword,
description: "Settings for media pool.",
children: [
%{
key: :max_connections,
type: :integer,
description: "Number workers in the pool.",
suggestions: [50]
},
%{
key: :timeout,
type: :integer,
description: "Timeout while `hackney` will wait for response.",
suggestions: [150_000]
}
]
},
%{
key: :upload,
type: :keyword,
description: "Settings for upload pool.",
children: [
%{
key: :max_connections,
type: :integer,
description: "Number workers in the pool.",
suggestions: [25]
},
%{
key: :timeout,
type: :integer,
description: "Timeout while `hackney` will wait for response.",
suggestions: [300_000]
}
]
}
]
},
%{ %{
group: :pleroma, group: :pleroma,
key: :restrict_unauthenticated, key: :restrict_unauthenticated,

View file

@ -90,6 +90,8 @@ config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock
config :pleroma, :modules, runtime_dir: "test/fixtures/modules" config :pleroma, :modules, runtime_dir: "test/fixtures/modules"
config :pleroma, Pleroma.Gun, Pleroma.GunMock
config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: true config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: true
config :pleroma, Pleroma.Plugs.RemoteIp, enabled: false config :pleroma, Pleroma.Plugs.RemoteIp, enabled: false

View file

@ -841,6 +841,8 @@ Some modifications are necessary to save the config settings correctly:
Most of the settings will be applied in `runtime`, this means that you don't need to restart the instance. But some settings are applied in `compile time` and require a reboot of the instance, such as: Most of the settings will be applied in `runtime`, this means that you don't need to restart the instance. But some settings are applied in `compile time` and require a reboot of the instance, such as:
- all settings inside these keys: - all settings inside these keys:
- `:hackney_pools` - `:hackney_pools`
- `:connections_pool`
- `:pools`
- `:chat` - `:chat`
- partially settings inside these keys: - partially settings inside these keys:
- `:seconds_valid` in `Pleroma.Captcha` - `:seconds_valid` in `Pleroma.Captcha`

View file

@ -369,8 +369,7 @@ Available caches:
* `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`) * `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`)
* `send_user_agent`: should we include a user agent with HTTP requests? (default: `true`) * `send_user_agent`: should we include a user agent with HTTP requests? (default: `true`)
* `user_agent`: what user agent should we use? (default: `:default`), must be string or `:default` * `user_agent`: what user agent should we use? (default: `:default`), must be string or `:default`
* `adapter`: array of hackney options * `adapter`: array of adapter options
### :hackney_pools ### :hackney_pools
@ -389,6 +388,42 @@ For each pool, the options are:
* `timeout` - retention duration for connections * `timeout` - retention duration for connections
### :connections_pool
*For `gun` adapter*
Advanced settings for connections pool. Pool with opened connections. These connections can be reused in worker pools.
For big instances it's recommended to increase `config :pleroma, :connections_pool, max_connections: 500` up to 500-1000.
It will increase memory usage, but federation would work faster.
* `:checkin_timeout` - timeout to checkin connection from pool. Default: 250ms.
* `:max_connections` - maximum number of connections in the pool. Default: 250 connections.
* `:retry` - number of retries, while `gun` will try to reconnect if connection goes down. Default: 1.
* `:retry_timeout` - time between retries when `gun` will try to reconnect in milliseconds. Default: 1000ms.
* `:await_up_timeout` - timeout while `gun` will wait until connection is up. Default: 5000ms.
### :pools
*For `gun` adapter*
Advanced settings for workers pools.
There are four pools used:
* `:federation` for the federation jobs.
You may want this pool max_connections to be at least equal to the number of federator jobs + retry queue jobs.
* `:media` for rich media, media proxy
* `:upload` for uploaded media (if using a remote uploader and `proxy_remote: true`)
* `:default` for other requests
For each pool, the options are:
* `:size` - how much workers the pool can hold
* `:timeout` - timeout while `gun` will wait for response
* `:max_overflow` - additional workers if pool is under load
## Captcha ## Captcha
### Pleroma.Captcha ### Pleroma.Captcha

View file

@ -75,4 +75,43 @@ defmodule Mix.Tasks.Pleroma.Benchmark do
inputs: inputs inputs: inputs
) )
end end
def run(["adapters"]) do
start_pleroma()
:ok =
Pleroma.Gun.Conn.open(
"https://httpbin.org/stream-bytes/1500",
:gun_connections
)
Process.sleep(1_500)
Benchee.run(
%{
"Without conn and without pool" => fn ->
{:ok, %Tesla.Env{}} =
Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
adapter: [pool: :no_pool, receive_conn: false]
)
end,
"Without conn and with pool" => fn ->
{:ok, %Tesla.Env{}} =
Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
adapter: [receive_conn: false]
)
end,
"With reused conn and without pool" => fn ->
{:ok, %Tesla.Env{}} =
Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
adapter: [pool: :no_pool]
)
end,
"With reused conn and with pool" => fn ->
{:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500")
end
},
parallel: 10
)
end
end end

View file

@ -4,13 +4,13 @@
defmodule Mix.Tasks.Pleroma.Emoji do defmodule Mix.Tasks.Pleroma.Emoji do
use Mix.Task use Mix.Task
import Mix.Pleroma
@shortdoc "Manages emoji packs" @shortdoc "Manages emoji packs"
@moduledoc File.read!("docs/administration/CLI_tasks/emoji.md") @moduledoc File.read!("docs/administration/CLI_tasks/emoji.md")
def run(["ls-packs" | args]) do def run(["ls-packs" | args]) do
Mix.Pleroma.start_pleroma() start_pleroma()
Application.ensure_all_started(:hackney)
{options, [], []} = parse_global_opts(args) {options, [], []} = parse_global_opts(args)
@ -36,8 +36,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
end end
def run(["get-packs" | args]) do def run(["get-packs" | args]) do
Mix.Pleroma.start_pleroma() start_pleroma()
Application.ensure_all_started(:hackney)
{options, pack_names, []} = parse_global_opts(args) {options, pack_names, []} = parse_global_opts(args)
@ -135,7 +134,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
end end
def run(["gen-pack", src]) do def run(["gen-pack", src]) do
Application.ensure_all_started(:hackney) start_pleroma()
proposed_name = Path.basename(src) |> Path.rootname() proposed_name = Path.basename(src) |> Path.rootname()
name = String.trim(IO.gets("Pack name [#{proposed_name}]: ")) name = String.trim(IO.gets("Pack name [#{proposed_name}]: "))

View file

@ -3,8 +3,12 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Application do defmodule Pleroma.Application do
import Cachex.Spec
use Application use Application
import Cachex.Spec
alias Pleroma.Config
require Logger require Logger
@name Mix.Project.config()[:name] @name Mix.Project.config()[:name]
@ -18,9 +22,9 @@ defmodule Pleroma.Application do
def repository, do: @repository def repository, do: @repository
def user_agent do def user_agent do
case Pleroma.Config.get([:http, :user_agent], :default) do case Config.get([:http, :user_agent], :default) do
:default -> :default ->
info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>" info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>"
named_version() <> "; " <> info named_version() <> "; " <> info
custom -> custom ->
@ -33,27 +37,51 @@ defmodule Pleroma.Application do
def start(_type, _args) do def start(_type, _args) do
Pleroma.Config.Holder.save_default() Pleroma.Config.Holder.save_default()
Pleroma.HTML.compile_scrubbers() Pleroma.HTML.compile_scrubbers()
Pleroma.Config.DeprecationWarnings.warn() Config.DeprecationWarnings.warn()
Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled() Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
Pleroma.Repo.check_migrations_applied!() Pleroma.Repo.check_migrations_applied!()
setup_instrumenters() setup_instrumenters()
load_custom_modules() load_custom_modules()
adapter = Application.get_env(:tesla, :adapter)
if adapter == Tesla.Adapter.Gun do
if version = Pleroma.OTPVersion.version() do
[major, minor] =
version
|> String.split(".")
|> Enum.map(&String.to_integer/1)
|> Enum.take(2)
if (major == 22 and minor < 2) or major < 22 do
raise "
!!!OTP VERSION WARNING!!!
You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains.
"
end
else
raise "
!!!OTP VERSION WARNING!!!
To support correct handling of unordered certificates chains - OTP version must be > 22.2.
"
end
end
# Define workers and child supervisors to be supervised # Define workers and child supervisors to be supervised
children = children =
[ [
Pleroma.Repo, Pleroma.Repo,
Pleroma.Config.TransferTask, Config.TransferTask,
Pleroma.Emoji, Pleroma.Emoji,
Pleroma.Captcha, Pleroma.Captcha,
Pleroma.Plugs.RateLimiter.Supervisor Pleroma.Plugs.RateLimiter.Supervisor
] ++ ] ++
cachex_children() ++ cachex_children() ++
hackney_pool_children() ++ http_children(adapter, @env) ++
[ [
Pleroma.Stats, Pleroma.Stats,
Pleroma.JobQueueMonitor, Pleroma.JobQueueMonitor,
{Oban, Pleroma.Config.get(Oban)} {Oban, Config.get(Oban)}
] ++ ] ++
task_children(@env) ++ task_children(@env) ++
streamer_child(@env) ++ streamer_child(@env) ++
@ -70,7 +98,7 @@ defmodule Pleroma.Application do
end end
def load_custom_modules do def load_custom_modules do
dir = Pleroma.Config.get([:modules, :runtime_dir]) dir = Config.get([:modules, :runtime_dir])
if dir && File.exists?(dir) do if dir && File.exists?(dir) do
dir dir
@ -111,20 +139,6 @@ defmodule Pleroma.Application do
Pleroma.Web.Endpoint.Instrumenter.setup() Pleroma.Web.Endpoint.Instrumenter.setup()
end end
def enabled_hackney_pools do
[:media] ++
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
[:federation]
else
[]
end ++
if Pleroma.Config.get([Pleroma.Upload, :proxy_remote]) do
[:upload]
else
[]
end
end
defp cachex_children do defp cachex_children do
[ [
build_cachex("used_captcha", ttl_interval: seconds_valid_interval()), build_cachex("used_captcha", ttl_interval: seconds_valid_interval()),
@ -146,7 +160,7 @@ defmodule Pleroma.Application do
do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60)) do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))
defp seconds_valid_interval, defp seconds_valid_interval,
do: :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid])) do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid]))
defp build_cachex(type, opts), defp build_cachex(type, opts),
do: %{ do: %{
@ -155,7 +169,7 @@ defmodule Pleroma.Application do
type: :worker type: :worker
} }
defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled]) defp chat_enabled?, do: Config.get([:chat, :enabled])
defp streamer_child(env) when env in [:test, :benchmark], do: [] defp streamer_child(env) when env in [:test, :benchmark], do: []
@ -169,13 +183,6 @@ defmodule Pleroma.Application do
defp chat_child(_, _), do: [] defp chat_child(_, _), do: []
defp hackney_pool_children do
for pool <- enabled_hackney_pools() do
options = Pleroma.Config.get([:hackney_pools, pool])
:hackney_pool.child_spec(pool, options)
end
end
defp task_children(:test) do defp task_children(:test) do
[ [
%{ %{
@ -200,4 +207,31 @@ defmodule Pleroma.Application do
} }
] ]
end end
# start hackney and gun pools in tests
defp http_children(_, :test) do
hackney_options = Config.get([:hackney_pools, :federation])
hackney_pool = :hackney_pool.child_spec(:federation, hackney_options)
[hackney_pool, Pleroma.Pool.Supervisor]
end
defp http_children(Tesla.Adapter.Hackney, _) do
pools = [:federation, :media]
pools =
if Config.get([Pleroma.Upload, :proxy_remote]) do
[:upload | pools]
else
pools
end
for pool <- pools do
options = Config.get([:hackney_pools, pool])
:hackney_pool.child_spec(pool, options)
end
end
defp http_children(Tesla.Adapter.Gun, _), do: [Pleroma.Pool.Supervisor]
defp http_children(_, _), do: []
end end

View file

@ -278,8 +278,6 @@ defmodule Pleroma.ConfigDB do
} }
end end
defp do_convert({:partial_chain, entity}), do: %{"tuple" => [":partial_chain", inspect(entity)]}
defp do_convert(entity) when is_tuple(entity) do defp do_convert(entity) when is_tuple(entity) do
value = value =
entity entity
@ -323,15 +321,6 @@ defmodule Pleroma.ConfigDB do
{:proxy_url, {do_transform_string(type), parse_host(host), port}} {:proxy_url, {do_transform_string(type), parse_host(host), port}}
end end
defp do_transform(%{"tuple" => [":partial_chain", entity]}) do
{partial_chain, []} =
entity
|> String.replace(~r/[^\w|^{:,[|^,|^[|^\]^}|^\/|^\.|^"]^\s/, "")
|> Code.eval_string()
{:partial_chain, partial_chain}
end
defp do_transform(%{"tuple" => entity}) do defp do_transform(%{"tuple" => entity}) do
Enum.reduce(entity, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end) Enum.reduce(entity, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end)
end end

View file

@ -5,6 +5,7 @@
defmodule Pleroma.Config.TransferTask do defmodule Pleroma.Config.TransferTask do
use Task use Task
alias Pleroma.Config
alias Pleroma.ConfigDB alias Pleroma.ConfigDB
alias Pleroma.Repo alias Pleroma.Repo
@ -18,7 +19,9 @@ defmodule Pleroma.Config.TransferTask do
{:pleroma, Oban}, {:pleroma, Oban},
{:pleroma, :rate_limit}, {:pleroma, :rate_limit},
{:pleroma, :markup}, {:pleroma, :markup},
{:plerome, :streamer} {:pleroma, :streamer},
{:pleroma, :pools},
{:pleroma, :connections_pool}
] ]
@reboot_time_subkeys [ @reboot_time_subkeys [
@ -32,45 +35,33 @@ defmodule Pleroma.Config.TransferTask do
{:pleroma, :gopher, [:enabled]} {:pleroma, :gopher, [:enabled]}
] ]
@reject [nil, :prometheus]
def start_link(_) do def start_link(_) do
load_and_update_env() load_and_update_env()
if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo) if Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo)
:ignore :ignore
end end
@spec load_and_update_env([ConfigDB.t()]) :: :ok | false @spec load_and_update_env([ConfigDB.t()], boolean()) :: :ok
def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do def load_and_update_env(deleted_settings \\ [], restart_pleroma? \\ true) do
with {:configurable, true} <- with {_, true} <- {:configurable, Config.get(:configurable_from_database)} do
{:configurable, Pleroma.Config.get(:configurable_from_database)},
true <- Ecto.Adapters.SQL.table_exists?(Repo, "config"),
started_applications <- Application.started_applications() do
# We need to restart applications for loaded settings take effect # We need to restart applications for loaded settings take effect
in_db = Repo.all(ConfigDB) # TODO: some problem with prometheus after restart!
reject_restart =
with_deleted = in_db ++ deleted if restart_pleroma? do
[nil, :prometheus]
reject_for_restart = if restart_pleroma?, do: @reject, else: [:pleroma | @reject]
applications =
with_deleted
|> Enum.map(&merge_and_update(&1))
|> Enum.uniq()
# TODO: some problem with prometheus after restart!
|> Enum.reject(&(&1 in reject_for_restart))
# to be ensured that pleroma will be restarted last
applications =
if :pleroma in applications do
List.delete(applications, :pleroma) ++ [:pleroma]
else else
Restarter.Pleroma.rebooted() [:pleroma, nil, :prometheus]
applications
end end
Enum.each(applications, &restart(started_applications, &1, Pleroma.Config.get(:env))) started_applications = Application.started_applications()
(Repo.all(ConfigDB) ++ deleted_settings)
|> Enum.map(&merge_and_update/1)
|> Enum.uniq()
|> Enum.reject(&(&1 in reject_restart))
|> maybe_set_pleroma_last()
|> Enum.each(&restart(started_applications, &1, Config.get(:env)))
:ok :ok
else else
@ -78,42 +69,54 @@ defmodule Pleroma.Config.TransferTask do
end end
end end
defp maybe_set_pleroma_last(apps) do
# to be ensured that pleroma will be restarted last
if :pleroma in apps do
apps
|> List.delete(:pleroma)
|> List.insert_at(-1, :pleroma)
else
Restarter.Pleroma.rebooted()
apps
end
end
defp group_for_restart(:logger, key, _, merged_value) do
# change logger configuration in runtime, without restart
if Keyword.keyword?(merged_value) and
key not in [:compile_time_application, :backends, :compile_time_purge_matching] do
Logger.configure_backend(key, merged_value)
else
Logger.configure([{key, merged_value}])
end
nil
end
defp group_for_restart(group, _, _, _) when group != :pleroma, do: group
defp group_for_restart(group, key, value, _) do
if pleroma_need_restart?(group, key, value), do: group
end
defp merge_and_update(setting) do defp merge_and_update(setting) do
try do try do
key = ConfigDB.from_string(setting.key) key = ConfigDB.from_string(setting.key)
group = ConfigDB.from_string(setting.group) group = ConfigDB.from_string(setting.group)
default = Pleroma.Config.Holder.default_config(group, key) default = Config.Holder.default_config(group, key)
value = ConfigDB.from_binary(setting.value) value = ConfigDB.from_binary(setting.value)
merged_value = merged_value =
if Ecto.get_meta(setting, :state) == :deleted do cond do
default Ecto.get_meta(setting, :state) == :deleted -> default
else can_be_merged?(default, value) -> ConfigDB.merge_group(group, key, default, value)
if can_be_merged?(default, value) do true -> value
ConfigDB.merge_group(group, key, default, value)
else
value
end
end end
:ok = update_env(group, key, merged_value) :ok = update_env(group, key, merged_value)
if group != :logger do group_for_restart(group, key, value, merged_value)
if group != :pleroma or pleroma_need_restart?(group, key, value) do
group
end
else
# change logger configuration in runtime, without restart
if Keyword.keyword?(merged_value) and
key not in [:compile_time_application, :backends, :compile_time_purge_matching] do
Logger.configure_backend(key, merged_value)
else
Logger.configure([{key, merged_value}])
end
nil
end
rescue rescue
error -> error ->
error_msg = error_msg =

45
lib/pleroma/gun/api.ex Normal file
View file

@ -0,0 +1,45 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Gun.API do
@behaviour Pleroma.Gun
alias Pleroma.Gun
@gun_keys [
:connect_timeout,
:http_opts,
:http2_opts,
:protocols,
:retry,
:retry_timeout,
:trace,
:transport,
:tls_opts,
:tcp_opts,
:socks_opts,
:ws_opts
]
@impl Gun
def open(host, port, opts \\ %{}), do: :gun.open(host, port, Map.take(opts, @gun_keys))
@impl Gun
defdelegate info(pid), to: :gun
@impl Gun
defdelegate close(pid), to: :gun
@impl Gun
defdelegate await_up(pid, timeout \\ 5_000), to: :gun
@impl Gun
defdelegate connect(pid, opts), to: :gun
@impl Gun
defdelegate await(pid, ref), to: :gun
@impl Gun
defdelegate set_owner(pid, owner), to: :gun
end

196
lib/pleroma/gun/conn.ex Normal file
View file

@ -0,0 +1,196 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Gun.Conn do
@moduledoc """
Struct for gun connection data
"""
alias Pleroma.Gun
alias Pleroma.Pool.Connections
require Logger
@type gun_state :: :up | :down
@type conn_state :: :active | :idle
@type t :: %__MODULE__{
conn: pid(),
gun_state: gun_state(),
conn_state: conn_state(),
used_by: [pid()],
last_reference: pos_integer(),
crf: float(),
retries: pos_integer()
}
defstruct conn: nil,
gun_state: :open,
conn_state: :init,
used_by: [],
last_reference: 0,
crf: 1,
retries: 0
@spec open(String.t() | URI.t(), atom(), keyword()) :: :ok | nil
def open(url, name, opts \\ [])
def open(url, name, opts) when is_binary(url), do: open(URI.parse(url), name, opts)
def open(%URI{} = uri, name, opts) do
pool_opts = Pleroma.Config.get([:connections_pool], [])
opts =
opts
|> Enum.into(%{})
|> Map.put_new(:retry, pool_opts[:retry] || 1)
|> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000)
|> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000)
|> maybe_add_tls_opts(uri)
key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
conn_pid =
if Connections.count(name) < opts[:max_connection] do
do_open(uri, opts)
else
close_least_used_and_do_open(name, uri, opts)
end
if is_pid(conn_pid) do
conn = %Pleroma.Gun.Conn{
conn: conn_pid,
gun_state: :up,
conn_state: :active,
last_reference: :os.system_time(:second)
}
:ok = Gun.set_owner(conn_pid, Process.whereis(name))
Connections.add_conn(name, key, conn)
end
end
defp maybe_add_tls_opts(opts, %URI{scheme: "http"}), do: opts
defp maybe_add_tls_opts(opts, %URI{scheme: "https", host: host}) do
tls_opts = [
verify: :verify_peer,
cacertfile: CAStore.file_path(),
depth: 20,
reuse_sessions: false,
verify_fun:
{&:ssl_verify_hostname.verify_fun/3,
[check_hostname: Pleroma.HTTP.Connection.format_host(host)]}
]
tls_opts =
if Keyword.keyword?(opts[:tls_opts]) do
Keyword.merge(tls_opts, opts[:tls_opts])
else
tls_opts
end
Map.put(opts, :tls_opts, tls_opts)
end
defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do
connect_opts =
uri
|> destination_opts()
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
with open_opts <- Map.delete(opts, :tls_opts),
{:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]),
stream <- Gun.connect(conn, connect_opts),
{:response, :fin, 200, _} <- Gun.await(conn, stream) do
conn
else
error ->
Logger.warn(
"Opening proxied connection to #{compose_uri_log(uri)} failed with error #{
inspect(error)
}"
)
error
end
end
defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do
version =
proxy_type
|> to_string()
|> String.last()
|> case do
"4" -> 4
_ -> 5
end
socks_opts =
uri
|> destination_opts()
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
|> Map.put(:version, version)
opts =
opts
|> Map.put(:protocols, [:socks])
|> Map.put(:socks_opts, socks_opts)
with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
conn
else
error ->
Logger.warn(
"Opening socks proxied connection to #{compose_uri_log(uri)} failed with error #{
inspect(error)
}"
)
error
end
end
defp do_open(%URI{host: host, port: port} = uri, opts) do
host = Pleroma.HTTP.Connection.parse_host(host)
with {:ok, conn} <- Gun.open(host, port, opts),
{:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
conn
else
error ->
Logger.warn(
"Opening connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}"
)
error
end
end
defp destination_opts(%URI{host: host, port: port}) do
host = Pleroma.HTTP.Connection.parse_host(host)
%{host: host, port: port}
end
defp add_http2_opts(opts, "https", tls_opts) do
Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts})
end
defp add_http2_opts(opts, _, _), do: opts
defp close_least_used_and_do_open(name, uri, opts) do
with [{key, conn} | _conns] <- Connections.get_unused_conns(name),
:ok <- Gun.close(conn.conn) do
Connections.remove_conn(name, key)
do_open(uri, opts)
else
[] -> {:error, :pool_overflowed}
end
end
def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do
"#{scheme}://#{host}#{path}"
end
end

31
lib/pleroma/gun/gun.ex Normal file
View file

@ -0,0 +1,31 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Gun do
@callback open(charlist(), pos_integer(), map()) :: {:ok, pid()}
@callback info(pid()) :: map()
@callback close(pid()) :: :ok
@callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()}
@callback connect(pid(), map()) :: reference()
@callback await(pid(), reference()) :: {:response, :fin, 200, []}
@callback set_owner(pid(), pid()) :: :ok
@api Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API)
defp api, do: @api
def open(host, port, opts), do: api().open(host, port, opts)
def info(pid), do: api().info(pid)
def close(pid), do: api().close(pid)
def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout)
def connect(pid, opts), do: api().connect(pid, opts)
def await(pid, ref), do: api().await(pid, ref)
def set_owner(pid, owner), do: api().set_owner(pid, owner)
end

View file

@ -0,0 +1,41 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.AdapterHelper do
alias Pleroma.HTTP.Connection
@type proxy ::
{Connection.host(), pos_integer()}
| {Connection.proxy_type(), Connection.host(), pos_integer()}
@callback options(keyword(), URI.t()) :: keyword()
@callback after_request(keyword()) :: :ok
@spec options(keyword(), URI.t()) :: keyword()
def options(opts, _uri) do
proxy = Pleroma.Config.get([:http, :proxy_url], nil)
maybe_add_proxy(opts, format_proxy(proxy))
end
@spec maybe_get_conn(URI.t(), keyword()) :: keyword()
def maybe_get_conn(_uri, opts), do: opts
@spec after_request(keyword()) :: :ok
def after_request(_opts), do: :ok
@spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil
def format_proxy(nil), do: nil
def format_proxy(proxy_url) do
case Connection.parse_proxy(proxy_url) do
{:ok, host, port} -> {host, port}
{:ok, type, host, port} -> {type, host, port}
_ -> nil
end
end
@spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword()
def maybe_add_proxy(opts, nil), do: opts
def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy)
end

View file

@ -0,0 +1,77 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.AdapterHelper.Gun do
@behaviour Pleroma.HTTP.AdapterHelper
alias Pleroma.HTTP.AdapterHelper
alias Pleroma.Pool.Connections
require Logger
@defaults [
connect_timeout: 5_000,
domain_lookup_timeout: 5_000,
tls_handshake_timeout: 5_000,
retry: 1,
retry_timeout: 1000,
await_up_timeout: 5_000
]
@spec options(keyword(), URI.t()) :: keyword()
def options(incoming_opts \\ [], %URI{} = uri) do
proxy =
Pleroma.Config.get([:http, :proxy_url])
|> AdapterHelper.format_proxy()
config_opts = Pleroma.Config.get([:http, :adapter], [])
@defaults
|> Keyword.merge(config_opts)
|> add_scheme_opts(uri)
|> AdapterHelper.maybe_add_proxy(proxy)
|> maybe_get_conn(uri, incoming_opts)
end
@spec after_request(keyword()) :: :ok
def after_request(opts) do
if opts[:conn] && opts[:body_as] != :chunks do
Connections.checkout(opts[:conn], self(), :gun_connections)
end
:ok
end
defp add_scheme_opts(opts, %{scheme: "http"}), do: opts
defp add_scheme_opts(opts, %{scheme: "https"}) do
opts
|> Keyword.put(:certificates_verification, true)
|> Keyword.put(:tls_opts, log_level: :warning)
end
defp maybe_get_conn(adapter_opts, uri, incoming_opts) do
{receive_conn?, opts} =
adapter_opts
|> Keyword.merge(incoming_opts)
|> Keyword.pop(:receive_conn, true)
if Connections.alive?(:gun_connections) and receive_conn? do
checkin_conn(uri, opts)
else
opts
end
end
defp checkin_conn(uri, opts) do
case Connections.checkin(uri, :gun_connections) do
nil ->
Task.start(Pleroma.Gun.Conn, :open, [uri, :gun_connections, opts])
opts
conn when is_pid(conn) ->
Keyword.merge(opts, conn: conn, close_conn: false)
end
end
end

View file

@ -0,0 +1,43 @@
defmodule Pleroma.HTTP.AdapterHelper.Hackney do
@behaviour Pleroma.HTTP.AdapterHelper
@defaults [
connect_timeout: 10_000,
recv_timeout: 20_000,
follow_redirect: true,
force_redirect: true,
pool: :federation
]
@spec options(keyword(), URI.t()) :: keyword()
def options(connection_opts \\ [], %URI{} = uri) do
proxy = Pleroma.Config.get([:http, :proxy_url])
config_opts = Pleroma.Config.get([:http, :adapter], [])
@defaults
|> Keyword.merge(config_opts)
|> Keyword.merge(connection_opts)
|> add_scheme_opts(uri)
|> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
end
defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts
defp add_scheme_opts(opts, %URI{scheme: "https", host: host}) do
ssl_opts = [
ssl_options: [
# Workaround for remote server certificate chain issues
partial_chain: &:hackney_connect.partial_chain/1,
# We don't support TLS v1.3 yet
versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"],
server_name_indication: to_charlist(host)
]
]
Keyword.merge(opts, ssl_opts)
end
def after_request(_), do: :ok
end

View file

@ -4,40 +4,121 @@
defmodule Pleroma.HTTP.Connection do defmodule Pleroma.HTTP.Connection do
@moduledoc """ @moduledoc """
Connection for http-requests. Configure Tesla.Client with default and customized adapter options.
""" """
@hackney_options [ alias Pleroma.Config
connect_timeout: 10_000, alias Pleroma.HTTP.AdapterHelper
recv_timeout: 20_000,
follow_redirect: true, require Logger
force_redirect: true,
pool: :federation @defaults [pool: :federation]
]
@adapter Application.get_env(:tesla, :adapter) @type ip_address :: ipv4_address() | ipv6_address()
@type ipv4_address :: {0..255, 0..255, 0..255, 0..255}
@type ipv6_address ::
{0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535}
@type proxy_type() :: :socks4 | :socks5
@type host() :: charlist() | ip_address()
@doc """ @doc """
Configure a client connection Merge default connection & adapter options with received ones.
# Returns
Tesla.Env.client
""" """
@spec new(Keyword.t()) :: Tesla.Env.client()
def new(opts \\ []) do @spec options(URI.t(), keyword()) :: keyword()
Tesla.client([], {@adapter, hackney_options(opts)}) def options(%URI{} = uri, opts \\ []) do
@defaults
|> pool_timeout()
|> Keyword.merge(opts)
|> adapter_helper().options(uri)
end end
# fetch Hackney options defp pool_timeout(opts) do
# {config_key, default} =
def hackney_options(opts) do if adapter() == Tesla.Adapter.Gun do
options = Keyword.get(opts, :adapter, []) {:pools, Config.get([:pools, :default, :timeout])}
adapter_options = Pleroma.Config.get([:http, :adapter], []) else
proxy_url = Pleroma.Config.get([:http, :proxy_url], nil) {:hackney_pools, 10_000}
end
@hackney_options timeout = Config.get([config_key, opts[:pool], :timeout], default)
|> Keyword.merge(adapter_options)
|> Keyword.merge(options) Keyword.merge(opts, timeout: timeout)
|> Keyword.merge(proxy: proxy_url) end
@spec after_request(keyword()) :: :ok
def after_request(opts), do: adapter_helper().after_request(opts)
defp adapter, do: Application.get_env(:tesla, :adapter)
defp adapter_helper do
case adapter() do
Tesla.Adapter.Gun -> AdapterHelper.Gun
Tesla.Adapter.Hackney -> AdapterHelper.Hackney
_ -> AdapterHelper
end
end
@spec parse_proxy(String.t() | tuple() | nil) ::
{:ok, host(), pos_integer()}
| {:ok, proxy_type(), host(), pos_integer()}
| {:error, atom()}
| nil
def parse_proxy(nil), do: nil
def parse_proxy(proxy) when is_binary(proxy) do
with [host, port] <- String.split(proxy, ":"),
{port, ""} <- Integer.parse(port) do
{:ok, parse_host(host), port}
else
{_, _} ->
Logger.warn("Parsing port failed #{inspect(proxy)}")
{:error, :invalid_proxy_port}
:error ->
Logger.warn("Parsing port failed #{inspect(proxy)}")
{:error, :invalid_proxy_port}
_ ->
Logger.warn("Parsing proxy failed #{inspect(proxy)}")
{:error, :invalid_proxy}
end
end
def parse_proxy(proxy) when is_tuple(proxy) do
with {type, host, port} <- proxy do
{:ok, type, parse_host(host), port}
else
_ ->
Logger.warn("Parsing proxy failed #{inspect(proxy)}")
{:error, :invalid_proxy}
end
end
@spec parse_host(String.t() | atom() | charlist()) :: charlist() | ip_address()
def parse_host(host) when is_list(host), do: host
def parse_host(host) when is_atom(host), do: to_charlist(host)
def parse_host(host) when is_binary(host) do
host = to_charlist(host)
case :inet.parse_address(host) do
{:error, :einval} -> host
{:ok, ip} -> ip
end
end
@spec format_host(String.t()) :: charlist()
def format_host(host) do
host_charlist = to_charlist(host)
case :inet.parse_address(host_charlist) do
{:error, :einval} ->
:idna.encode(host_charlist)
{:ok, _ip} ->
host_charlist
end
end end
end end

View file

@ -4,21 +4,47 @@
defmodule Pleroma.HTTP do defmodule Pleroma.HTTP do
@moduledoc """ @moduledoc """
Wrapper for `Tesla.request/2`.
""" """
alias Pleroma.HTTP.Connection alias Pleroma.HTTP.Connection
alias Pleroma.HTTP.Request
alias Pleroma.HTTP.RequestBuilder, as: Builder alias Pleroma.HTTP.RequestBuilder, as: Builder
alias Tesla.Client
alias Tesla.Env
require Logger
@type t :: __MODULE__ @type t :: __MODULE__
@doc """ @doc """
Builds and perform http request. Performs GET request.
See `Pleroma.HTTP.request/5`
"""
@spec get(Request.url() | nil, Request.headers(), keyword()) ::
nil | {:ok, Env.t()} | {:error, any()}
def get(url, headers \\ [], options \\ [])
def get(nil, _, _), do: nil
def get(url, headers, options), do: request(:get, url, "", headers, options)
@doc """
Performs POST request.
See `Pleroma.HTTP.request/5`
"""
@spec post(Request.url(), String.t(), Request.headers(), keyword()) ::
{:ok, Env.t()} | {:error, any()}
def post(url, body, headers \\ [], options \\ []),
do: request(:post, url, body, headers, options)
@doc """
Builds and performs http request.
# Arguments: # Arguments:
`method` - :get, :post, :put, :delete `method` - :get, :post, :put, :delete
`url` `url` - full url
`body` `body` - request body
`headers` - a keyworld list of headers, e.g. `[{"content-type", "text/plain"}]` `headers` - a keyworld list of headers, e.g. `[{"content-type", "text/plain"}]`
`options` - custom, per-request middleware or adapter options `options` - custom, per-request middleware or adapter options
@ -26,61 +52,66 @@ defmodule Pleroma.HTTP do
`{:ok, %Tesla.Env{}}` or `{:error, error}` `{:ok, %Tesla.Env{}}` or `{:error, error}`
""" """
def request(method, url, body \\ "", headers \\ [], options \\ []) do @spec request(atom(), Request.url(), String.t(), Request.headers(), keyword()) ::
try do {:ok, Env.t()} | {:error, any()}
options = def request(method, url, body, headers, options) when is_binary(url) do
process_request_options(options)
|> process_sni_options(url)
params = Keyword.get(options, :params, [])
%{}
|> Builder.method(method)
|> Builder.headers(headers)
|> Builder.opts(options)
|> Builder.url(url)
|> Builder.add_param(:body, :body, body)
|> Builder.add_param(:query, :query, params)
|> Enum.into([])
|> (&Tesla.request(Connection.new(options), &1)).()
rescue
e ->
{:error, e}
catch
:exit, e ->
{:error, e}
end
end
defp process_sni_options(options, nil), do: options
defp process_sni_options(options, url) do
uri = URI.parse(url) uri = URI.parse(url)
host = uri.host |> to_charlist() adapter_opts = Connection.options(uri, options[:adapter] || [])
options = put_in(options[:adapter], adapter_opts)
params = options[:params] || []
request = build_request(method, headers, options, url, body, params)
case uri.scheme do adapter = Application.get_env(:tesla, :adapter)
"https" -> options ++ [ssl: [server_name_indication: host]] client = Tesla.client([Tesla.Middleware.FollowRedirects], adapter)
_ -> options
end pid = Process.whereis(adapter_opts[:pool])
pool_alive? =
if adapter == Tesla.Adapter.Gun && pid do
Process.alive?(pid)
else
false
end
request_opts =
adapter_opts
|> Enum.into(%{})
|> Map.put(:env, Pleroma.Config.get([:env]))
|> Map.put(:pool_alive?, pool_alive?)
response = request(client, request, request_opts)
Connection.after_request(adapter_opts)
response
end end
def process_request_options(options) do @spec request(Client.t(), keyword(), map()) :: {:ok, Env.t()} | {:error, any()}
Keyword.merge(Pleroma.HTTP.Connection.hackney_options([]), options) def request(%Client{} = client, request, %{env: :test}), do: request(client, request)
def request(%Client{} = client, request, %{body_as: :chunks}), do: request(client, request)
def request(%Client{} = client, request, %{pool_alive?: false}), do: request(client, request)
def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do
:poolboy.transaction(
pool,
&Pleroma.Pool.Request.execute(&1, client, request, timeout),
timeout
)
end end
@doc """ @spec request(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()}
Performs GET request. def request(client, request), do: Tesla.request(client, request)
See `Pleroma.HTTP.request/5` defp build_request(method, headers, options, url, body, params) do
""" Builder.new()
def get(url, headers \\ [], options \\ []), |> Builder.method(method)
do: request(:get, url, "", headers, options) |> Builder.headers(headers)
|> Builder.opts(options)
@doc """ |> Builder.url(url)
Performs POST request. |> Builder.add_param(:body, :body, body)
|> Builder.add_param(:query, :query, params)
See `Pleroma.HTTP.request/5` |> Builder.convert_to_keyword()
""" end
def post(url, body, headers \\ [], options \\ []),
do: request(:post, url, body, headers, options)
end end

View file

@ -0,0 +1,23 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.Request do
@moduledoc """
Request struct.
"""
defstruct method: :get, url: "", query: [], headers: [], body: "", opts: []
@type method :: :head | :get | :delete | :trace | :options | :post | :put | :patch
@type url :: String.t()
@type headers :: [{String.t(), String.t()}]
@type t :: %__MODULE__{
method: method(),
url: url(),
query: keyword(),
headers: headers(),
body: String.t(),
opts: keyword()
}
end

View file

@ -7,136 +7,87 @@ defmodule Pleroma.HTTP.RequestBuilder do
Helper functions for building Tesla requests Helper functions for building Tesla requests
""" """
alias Pleroma.HTTP.Request
alias Tesla.Multipart
@doc """ @doc """
Specify the request method when building a request Creates new request
## Parameters
- request (Map) - Collected request options
- m (atom) - Request method
## Returns
Map
""" """
@spec method(map(), atom) :: map() @spec new(Request.t()) :: Request.t()
def method(request, m) do def new(%Request{} = request \\ %Request{}), do: request
Map.put_new(request, :method, m)
end
@doc """ @doc """
Specify the request method when building a request Specify the request method when building a request
## Parameters
- request (Map) - Collected request options
- u (String) - Request URL
## Returns
Map
""" """
@spec url(map(), String.t()) :: map() @spec method(Request.t(), Request.method()) :: Request.t()
def url(request, u) do def method(request, m), do: %{request | method: m}
Map.put_new(request, :url, u)
end @doc """
Specify the request method when building a request
"""
@spec url(Request.t(), Request.url()) :: Request.t()
def url(request, u), do: %{request | url: u}
@doc """ @doc """
Add headers to the request Add headers to the request
""" """
@spec headers(map(), list(tuple)) :: map() @spec headers(Request.t(), Request.headers()) :: Request.t()
def headers(request, header_list) do def headers(request, headers) do
header_list = headers_list =
if Pleroma.Config.get([:http, :send_user_agent]) do if Pleroma.Config.get([:http, :send_user_agent]) do
header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}] [{"user-agent", Pleroma.Application.user_agent()} | headers]
else else
header_list headers
end end
Map.put_new(request, :headers, header_list) %{request | headers: headers_list}
end end
@doc """ @doc """
Add custom, per-request middleware or adapter options to the request Add custom, per-request middleware or adapter options to the request
""" """
@spec opts(map(), Keyword.t()) :: map() @spec opts(Request.t(), keyword()) :: Request.t()
def opts(request, options) do def opts(request, options), do: %{request | opts: options}
Map.put_new(request, :opts, options)
end
@doc """ @doc """
Add optional parameters to the request Add optional parameters to the request
## Parameters
- request (Map) - Collected request options
- definitions (Map) - Map of parameter name to parameter location.
- options (KeywordList) - The provided optional parameters
## Returns
Map
""" """
@spec add_optional_params(map(), %{optional(atom) => atom}, keyword()) :: map() @spec add_param(Request.t(), atom(), atom(), any()) :: Request.t()
def add_optional_params(request, _, []), do: request def add_param(request, :query, :query, values), do: %{request | query: values}
def add_optional_params(request, definitions, [{key, value} | tail]) do def add_param(request, :body, :body, value), do: %{request | body: value}
case definitions do
%{^key => location} ->
request
|> add_param(location, key, value)
|> add_optional_params(definitions, tail)
_ ->
add_optional_params(request, definitions, tail)
end
end
@doc """
Add optional parameters to the request
## Parameters
- request (Map) - Collected request options
- location (atom) - Where to put the parameter
- key (atom) - The name of the parameter
- value (any) - The value of the parameter
## Returns
Map
"""
@spec add_param(map(), atom, atom, any()) :: map()
def add_param(request, :query, :query, values), do: Map.put(request, :query, values)
def add_param(request, :body, :body, value), do: Map.put(request, :body, value)
def add_param(request, :body, key, value) do def add_param(request, :body, key, value) do
request request
|> Map.put_new_lazy(:body, &Tesla.Multipart.new/0) |> Map.put(:body, Multipart.new())
|> Map.update!( |> Map.update!(
:body, :body,
&Tesla.Multipart.add_field( &Multipart.add_field(
&1, &1,
key, key,
Jason.encode!(value), Jason.encode!(value),
headers: [{:"Content-Type", "application/json"}] headers: [{"content-type", "application/json"}]
) )
) )
end end
def add_param(request, :file, name, path) do def add_param(request, :file, name, path) do
request request
|> Map.put_new_lazy(:body, &Tesla.Multipart.new/0) |> Map.put(:body, Multipart.new())
|> Map.update!(:body, &Tesla.Multipart.add_file(&1, path, name: name)) |> Map.update!(:body, &Multipart.add_file(&1, path, name: name))
end end
def add_param(request, :form, name, value) do def add_param(request, :form, name, value) do
request Map.update(request, :body, %{name => value}, &Map.put(&1, name, value))
|> Map.update(:body, %{name => value}, &Map.put(&1, name, value))
end end
def add_param(request, location, key, value) do def add_param(request, location, key, value) do
Map.update(request, location, [{key, value}], &(&1 ++ [{key, value}])) Map.update(request, location, [{key, value}], &(&1 ++ [{key, value}]))
end end
def convert_to_keyword(request) do
request
|> Map.from_struct()
|> Enum.into([])
end
end end

View file

@ -141,7 +141,7 @@ defmodule Pleroma.Object.Fetcher do
date: date date: date
}) })
[{:Signature, signature}] [{"signature", signature}]
end end
defp sign_fetch(headers, id, date) do defp sign_fetch(headers, id, date) do
@ -154,7 +154,7 @@ defmodule Pleroma.Object.Fetcher do
defp maybe_date_fetch(headers, date) do defp maybe_date_fetch(headers, date) do
if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
headers ++ [{:Date, date}] headers ++ [{"date", date}]
else else
headers headers
end end
@ -166,7 +166,7 @@ defmodule Pleroma.Object.Fetcher do
date = Pleroma.Signature.signed_date() date = Pleroma.Signature.signed_date()
headers = headers =
[{:Accept, "application/activity+json"}] [{"accept", "application/activity+json"}]
|> maybe_date_fetch(date) |> maybe_date_fetch(date)
|> sign_fetch(id, date) |> sign_fetch(id, date)

View file

@ -0,0 +1,28 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.OTPVersion do
@spec version() :: String.t() | nil
def version do
# OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version
[
Path.join(:code.root_dir(), "OTP_VERSION"),
Path.join([:code.root_dir(), "releases", :erlang.system_info(:otp_release), "OTP_VERSION"])
]
|> get_version_from_files()
end
@spec get_version_from_files([Path.t()]) :: String.t() | nil
def get_version_from_files([]), do: nil
def get_version_from_files([path | paths]) do
if File.exists?(path) do
path
|> File.read!()
|> String.replace(~r/\r|\n|\s/, "")
else
get_version_from_files(paths)
end
end
end

View file

@ -0,0 +1,283 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Pool.Connections do
use GenServer
alias Pleroma.Config
alias Pleroma.Gun
require Logger
@type domain :: String.t()
@type conn :: Pleroma.Gun.Conn.t()
@type t :: %__MODULE__{
conns: %{domain() => conn()},
opts: keyword()
}
defstruct conns: %{}, opts: []
@spec start_link({atom(), keyword()}) :: {:ok, pid()}
def start_link({name, opts}) do
GenServer.start_link(__MODULE__, opts, name: name)
end
@impl true
def init(opts), do: {:ok, %__MODULE__{conns: %{}, opts: opts}}
@spec checkin(String.t() | URI.t(), atom()) :: pid() | nil
def checkin(url, name)
def checkin(url, name) when is_binary(url), do: checkin(URI.parse(url), name)
def checkin(%URI{} = uri, name) do
timeout = Config.get([:connections_pool, :checkin_timeout], 250)
GenServer.call(name, {:checkin, uri}, timeout)
end
@spec alive?(atom()) :: boolean()
def alive?(name) do
if pid = Process.whereis(name) do
Process.alive?(pid)
else
false
end
end
@spec get_state(atom()) :: t()
def get_state(name) do
GenServer.call(name, :state)
end
@spec count(atom()) :: pos_integer()
def count(name) do
GenServer.call(name, :count)
end
@spec get_unused_conns(atom()) :: [{domain(), conn()}]
def get_unused_conns(name) do
GenServer.call(name, :unused_conns)
end
@spec checkout(pid(), pid(), atom()) :: :ok
def checkout(conn, pid, name) do
GenServer.cast(name, {:checkout, conn, pid})
end
@spec add_conn(atom(), String.t(), Pleroma.Gun.Conn.t()) :: :ok
def add_conn(name, key, conn) do
GenServer.cast(name, {:add_conn, key, conn})
end
@spec remove_conn(atom(), String.t()) :: :ok
def remove_conn(name, key) do
GenServer.cast(name, {:remove_conn, key})
end
@impl true
def handle_cast({:add_conn, key, conn}, state) do
state = put_in(state.conns[key], conn)
Process.monitor(conn.conn)
{:noreply, state}
end
@impl true
def handle_cast({:checkout, conn_pid, pid}, state) do
state =
with true <- Process.alive?(conn_pid),
{key, conn} <- find_conn(state.conns, conn_pid),
used_by <- List.keydelete(conn.used_by, pid, 0) do
conn_state = if used_by == [], do: :idle, else: conn.conn_state
put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by})
else
false ->
Logger.debug("checkout for closed conn #{inspect(conn_pid)}")
state
nil ->
Logger.debug("checkout for alive conn #{inspect(conn_pid)}, but is not in state")
state
end
{:noreply, state}
end
@impl true
def handle_cast({:remove_conn, key}, state) do
state = put_in(state.conns, Map.delete(state.conns, key))
{:noreply, state}
end
@impl true
def handle_call({:checkin, uri}, from, state) do
key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
case state.conns[key] do
%{conn: pid, gun_state: :up} = conn ->
time = :os.system_time(:second)
last_reference = time - conn.last_reference
crf = crf(last_reference, 100, conn.crf)
state =
put_in(state.conns[key], %{
conn
| last_reference: time,
crf: crf,
conn_state: :active,
used_by: [from | conn.used_by]
})
{:reply, pid, state}
%{gun_state: :down} ->
{:reply, nil, state}
nil ->
{:reply, nil, state}
end
end
@impl true
def handle_call(:state, _from, state), do: {:reply, state, state}
@impl true
def handle_call(:count, _from, state) do
{:reply, Enum.count(state.conns), state}
end
@impl true
def handle_call(:unused_conns, _from, state) do
unused_conns =
state.conns
|> Enum.filter(&filter_conns/1)
|> Enum.sort(&sort_conns/2)
{:reply, unused_conns, state}
end
defp filter_conns({_, %{conn_state: :idle, used_by: []}}), do: true
defp filter_conns(_), do: false
defp sort_conns({_, c1}, {_, c2}) do
c1.crf <= c2.crf and c1.last_reference <= c2.last_reference
end
@impl true
def handle_info({:gun_up, conn_pid, _protocol}, state) do
%{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid)
host =
case :inet.ntoa(host) do
{:error, :einval} -> host
ip -> ip
end
key = "#{scheme}:#{host}:#{port}"
state =
with {key, conn} <- find_conn(state.conns, conn_pid, key),
{true, key} <- {Process.alive?(conn_pid), key} do
put_in(state.conns[key], %{
conn
| gun_state: :up,
conn_state: :active,
retries: 0
})
else
{false, key} ->
put_in(
state.conns,
Map.delete(state.conns, key)
)
nil ->
:ok = Gun.close(conn_pid)
state
end
{:noreply, state}
end
@impl true
def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do
retries = Config.get([:connections_pool, :retry], 1)
# we can't get info on this pid, because pid is dead
state =
with {key, conn} <- find_conn(state.conns, conn_pid),
{true, key} <- {Process.alive?(conn_pid), key} do
if conn.retries == retries do
:ok = Gun.close(conn.conn)
put_in(
state.conns,
Map.delete(state.conns, key)
)
else
put_in(state.conns[key], %{
conn
| gun_state: :down,
retries: conn.retries + 1
})
end
else
{false, key} ->
put_in(
state.conns,
Map.delete(state.conns, key)
)
nil ->
Logger.debug(":gun_down for conn which isn't found in state")
state
end
{:noreply, state}
end
@impl true
def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do
Logger.debug("received DOWM message for #{inspect(conn_pid)} reason -> #{inspect(reason)}")
state =
with {key, conn} <- find_conn(state.conns, conn_pid) do
Enum.each(conn.used_by, fn {pid, _ref} ->
Process.exit(pid, reason)
end)
put_in(
state.conns,
Map.delete(state.conns, key)
)
else
nil ->
Logger.debug(":DOWN for conn which isn't found in state")
state
end
{:noreply, state}
end
defp find_conn(conns, conn_pid) do
Enum.find(conns, fn {_key, conn} ->
conn.conn == conn_pid
end)
end
defp find_conn(conns, conn_pid, conn_key) do
Enum.find(conns, fn {key, conn} ->
key == conn_key and conn.conn == conn_pid
end)
end
def crf(current, steps, crf) do
1 + :math.pow(0.5, current / steps) * crf
end
end

22
lib/pleroma/pool/pool.ex Normal file
View file

@ -0,0 +1,22 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Pool do
def child_spec(opts) do
poolboy_opts =
opts
|> Keyword.put(:worker_module, Pleroma.Pool.Request)
|> Keyword.put(:name, {:local, opts[:name]})
|> Keyword.put(:size, opts[:size])
|> Keyword.put(:max_overflow, opts[:max_overflow])
%{
id: opts[:id] || {__MODULE__, make_ref()},
start: {:poolboy, :start_link, [poolboy_opts, [name: opts[:name]]]},
restart: :permanent,
shutdown: 5000,
type: :worker
}
end
end

View file

@ -0,0 +1,65 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Pool.Request do
use GenServer
require Logger
def start_link(args) do
GenServer.start_link(__MODULE__, args)
end
@impl true
def init(_), do: {:ok, []}
@spec execute(pid() | atom(), Tesla.Client.t(), keyword(), pos_integer()) ::
{:ok, Tesla.Env.t()} | {:error, any()}
def execute(pid, client, request, timeout) do
GenServer.call(pid, {:execute, client, request}, timeout)
end
@impl true
def handle_call({:execute, client, request}, _from, state) do
response = Pleroma.HTTP.request(client, request)
{:reply, response, state}
end
@impl true
def handle_info({:gun_data, _conn, _stream, _, _}, state) do
{:noreply, state}
end
@impl true
def handle_info({:gun_up, _conn, _protocol}, state) do
{:noreply, state}
end
@impl true
def handle_info({:gun_down, _conn, _protocol, _reason, _killed}, state) do
{:noreply, state}
end
@impl true
def handle_info({:gun_error, _conn, _stream, _error}, state) do
{:noreply, state}
end
@impl true
def handle_info({:gun_push, _conn, _stream, _new_stream, _method, _uri, _headers}, state) do
{:noreply, state}
end
@impl true
def handle_info({:gun_response, _conn, _stream, _, _status, _headers}, state) do
{:noreply, state}
end
@impl true
def handle_info(msg, state) do
Logger.warn("Received unexpected message #{inspect(__MODULE__)} #{inspect(msg)}")
{:noreply, state}
end
end

View file

@ -0,0 +1,42 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Pool.Supervisor do
use Supervisor
alias Pleroma.Config
alias Pleroma.Pool
def start_link(args) do
Supervisor.start_link(__MODULE__, args, name: __MODULE__)
end
def init(_) do
conns_child = %{
id: Pool.Connections,
start:
{Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]}
}
Supervisor.init([conns_child | pools()], strategy: :one_for_one)
end
defp pools do
pools = Config.get(:pools)
pools =
if Config.get([Pleroma.Upload, :proxy_remote]) == false do
Keyword.delete(pools, :upload)
else
pools
end
for {pool_name, pool_opts} <- pools do
pool_opts
|> Keyword.put(:id, {Pool, pool_name})
|> Keyword.put(:name, pool_name)
|> Pool.child_spec()
end
end
end

View file

@ -3,19 +3,23 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ReverseProxy.Client do defmodule Pleroma.ReverseProxy.Client do
@callback request(atom(), String.t(), [tuple()], String.t(), list()) :: @type status :: pos_integer()
{:ok, pos_integer(), [tuple()], reference() | map()} @type header_name :: String.t()
| {:ok, pos_integer(), [tuple()]} @type header_value :: String.t()
@type headers :: [{header_name(), header_value()}]
@callback request(atom(), String.t(), headers(), String.t(), list()) ::
{:ok, status(), headers(), reference() | map()}
| {:ok, status(), headers()}
| {:ok, reference()} | {:ok, reference()}
| {:error, term()} | {:error, term()}
@callback stream_body(reference() | pid() | map()) :: @callback stream_body(map()) :: {:ok, binary(), map()} | :done | {:error, atom() | String.t()}
{:ok, binary()} | :done | {:error, String.t()}
@callback close(reference() | pid() | map()) :: :ok @callback close(reference() | pid() | map()) :: :ok
def request(method, url, headers, "", opts \\ []) do def request(method, url, headers, body \\ "", opts \\ []) do
client().request(method, url, headers, "", opts) client().request(method, url, headers, body, opts)
end end
def stream_body(ref), do: client().stream_body(ref) def stream_body(ref), do: client().stream_body(ref)
@ -23,6 +27,12 @@ defmodule Pleroma.ReverseProxy.Client do
def close(ref), do: client().close(ref) def close(ref), do: client().close(ref)
defp client do defp client do
Pleroma.Config.get([Pleroma.ReverseProxy.Client], :hackney) :tesla
|> Application.get_env(:adapter)
|> client()
end end
defp client(Tesla.Adapter.Hackney), do: Pleroma.ReverseProxy.Client.Hackney
defp client(Tesla.Adapter.Gun), do: Pleroma.ReverseProxy.Client.Tesla
defp client(_), do: Pleroma.Config.get!(Pleroma.ReverseProxy.Client)
end end

View file

@ -0,0 +1,24 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ReverseProxy.Client.Hackney do
@behaviour Pleroma.ReverseProxy.Client
@impl true
def request(method, url, headers, body, opts \\ []) do
:hackney.request(method, url, headers, body, opts)
end
@impl true
def stream_body(ref) do
case :hackney.stream_body(ref) do
:done -> :done
{:ok, data} -> {:ok, data, ref}
{:error, error} -> {:error, error}
end
end
@impl true
def close(ref), do: :hackney.close(ref)
end

View file

@ -0,0 +1,90 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ReverseProxy.Client.Tesla do
@behaviour Pleroma.ReverseProxy.Client
@type headers() :: [{String.t(), String.t()}]
@type status() :: pos_integer()
@spec request(atom(), String.t(), headers(), String.t(), keyword()) ::
{:ok, status(), headers}
| {:ok, status(), headers, map()}
| {:error, atom() | String.t()}
| no_return()
@impl true
def request(method, url, headers, body, opts \\ []) do
check_adapter()
opts = Keyword.put(opts, :body_as, :chunks)
with {:ok, response} <-
Pleroma.HTTP.request(
method,
url,
body,
headers,
Keyword.put(opts, :adapter, opts)
) do
if is_map(response.body) and method != :head do
{:ok, response.status, response.headers, response.body}
else
{:ok, response.status, response.headers}
end
else
{:error, error} -> {:error, error}
end
end
@impl true
@spec stream_body(map()) ::
{:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return()
def stream_body(%{pid: pid, opts: opts, fin: true}) do
# if connection was reused, but in tesla were redirects,
# tesla returns new opened connection, which must be closed manually
if opts[:old_conn], do: Tesla.Adapter.Gun.close(pid)
# if there were redirects we need to checkout old conn
conn = opts[:old_conn] || opts[:conn]
if conn, do: :ok = Pleroma.Pool.Connections.checkout(conn, self(), :gun_connections)
:done
end
def stream_body(client) do
case read_chunk!(client) do
{:fin, body} ->
{:ok, body, Map.put(client, :fin, true)}
{:nofin, part} ->
{:ok, part, client}
{:error, error} ->
{:error, error}
end
end
defp read_chunk!(%{pid: pid, stream: stream, opts: opts}) do
adapter = check_adapter()
adapter.read_chunk(pid, stream, opts)
end
@impl true
@spec close(map) :: :ok | no_return()
def close(%{pid: pid}) do
adapter = check_adapter()
adapter.close(pid)
end
defp check_adapter do
adapter = Application.get_env(:tesla, :adapter)
unless adapter == Tesla.Adapter.Gun do
raise "#{adapter} doesn't support reading body in chunks"
end
adapter
end
end

View file

@ -3,8 +3,6 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ReverseProxy do defmodule Pleroma.ReverseProxy do
alias Pleroma.HTTP
@keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++ @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++
~w(if-unmodified-since if-none-match if-range range) ~w(if-unmodified-since if-none-match if-range range)
@resp_cache_headers ~w(etag date last-modified) @resp_cache_headers ~w(etag date last-modified)
@ -58,10 +56,10 @@ defmodule Pleroma.ReverseProxy do
* `req_headers`, `resp_headers` additional headers. * `req_headers`, `resp_headers` additional headers.
* `http`: options for [hackney](https://github.com/benoitc/hackney). * `http`: options for [hackney](https://github.com/benoitc/hackney) or [gun](https://github.com/ninenines/gun).
""" """
@default_hackney_options [pool: :media] @default_options [pool: :media]
@inline_content_types [ @inline_content_types [
"image/gif", "image/gif",
@ -94,11 +92,7 @@ defmodule Pleroma.ReverseProxy do
def call(_conn, _url, _opts \\ []) def call(_conn, _url, _opts \\ [])
def call(conn = %{method: method}, url, opts) when method in @methods do def call(conn = %{method: method}, url, opts) when method in @methods do
hackney_opts = client_opts = Keyword.merge(@default_options, Keyword.get(opts, :http, []))
Pleroma.HTTP.Connection.hackney_options([])
|> Keyword.merge(@default_hackney_options)
|> Keyword.merge(Keyword.get(opts, :http, []))
|> HTTP.process_request_options()
req_headers = build_req_headers(conn.req_headers, opts) req_headers = build_req_headers(conn.req_headers, opts)
@ -110,7 +104,7 @@ defmodule Pleroma.ReverseProxy do
end end
with {:ok, nil} <- Cachex.get(:failed_proxy_url_cache, url), with {:ok, nil} <- Cachex.get(:failed_proxy_url_cache, url),
{:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts), {:ok, code, headers, client} <- request(method, url, req_headers, client_opts),
:ok <- :ok <-
header_length_constraint( header_length_constraint(
headers, headers,
@ -156,11 +150,11 @@ defmodule Pleroma.ReverseProxy do
|> halt() |> halt()
end end
defp request(method, url, headers, hackney_opts) do defp request(method, url, headers, opts) do
Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}") Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}")
method = method |> String.downcase() |> String.to_existing_atom() method = method |> String.downcase() |> String.to_existing_atom()
case client().request(method, url, headers, "", hackney_opts) do case client().request(method, url, headers, "", opts) do
{:ok, code, headers, client} when code in @valid_resp_codes -> {:ok, code, headers, client} when code in @valid_resp_codes ->
{:ok, code, downcase_headers(headers), client} {:ok, code, downcase_headers(headers), client}
@ -210,7 +204,7 @@ defmodule Pleroma.ReverseProxy do
duration, duration,
Keyword.get(opts, :max_read_duration, @max_read_duration) Keyword.get(opts, :max_read_duration, @max_read_duration)
), ),
{:ok, data} <- client().stream_body(client), {:ok, data, client} <- client().stream_body(client),
{:ok, duration} <- increase_read_duration(duration), {:ok, duration} <- increase_read_duration(duration),
sent_so_far = sent_so_far + byte_size(data), sent_so_far = sent_so_far + byte_size(data),
:ok <- :ok <-

View file

@ -1,5 +1,5 @@
# Pleroma: A lightweight social networking server # Pleroma: A lightweight social networking server
# Copyright © 2019 Pleroma Authors <https://pleroma.social/> # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do

View file

@ -12,17 +12,23 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
require Logger require Logger
@hackney_options [ @options [
pool: :media, pool: :media
recv_timeout: 10_000
] ]
def perform(:prefetch, url) do def perform(:prefetch, url) do
Logger.debug("Prefetching #{inspect(url)}") Logger.debug("Prefetching #{inspect(url)}")
opts =
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
Keyword.put(@options, :recv_timeout, 10_000)
else
@options
end
url url
|> MediaProxy.url() |> MediaProxy.url()
|> HTTP.get([], adapter: @hackney_options) |> HTTP.get([], adapter: opts)
end end
def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do

View file

@ -1,5 +1,5 @@
# Pleroma: A lightweight social networking server # Pleroma: A lightweight social networking server
# Copyright © 2019 Pleroma Authors <https://pleroma.social/> # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do

View file

@ -3,11 +3,9 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RelMe do defmodule Pleroma.Web.RelMe do
@hackney_options [ @options [
pool: :media, pool: :media,
recv_timeout: 2_000, max_body: 2_000_000
max_body: 2_000_000,
with_body: true
] ]
if Pleroma.Config.get(:env) == :test do if Pleroma.Config.get(:env) == :test do
@ -25,8 +23,18 @@ defmodule Pleroma.Web.RelMe do
def parse(_), do: {:error, "No URL provided"} def parse(_), do: {:error, "No URL provided"}
defp parse_url(url) do defp parse_url(url) do
opts =
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
Keyword.merge(@options,
recv_timeout: 2_000,
with_body: true
)
else
@options
end
with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <- with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <-
Pleroma.HTTP.get(url, [], adapter: @hackney_options), Pleroma.HTTP.get(url, [], adapter: opts),
{:ok, html_tree} <- Floki.parse_document(html), {:ok, html_tree} <- Floki.parse_document(html),
data <- data <-
Floki.attribute(html_tree, "link[rel~=me]", "href") ++ Floki.attribute(html_tree, "link[rel~=me]", "href") ++

View file

@ -3,11 +3,9 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parser do defmodule Pleroma.Web.RichMedia.Parser do
@hackney_options [ @options [
pool: :media, pool: :media,
recv_timeout: 2_000, max_body: 2_000_000
max_body: 2_000_000,
with_body: true
] ]
defp parsers do defp parsers do
@ -77,8 +75,18 @@ defmodule Pleroma.Web.RichMedia.Parser do
end end
defp parse_url(url) do defp parse_url(url) do
opts =
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
Keyword.merge(@options,
recv_timeout: 2_000,
with_body: true
)
else
@options
end
try do try do
{:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options) {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: opts)
html html
|> parse_html() |> parse_html()

View file

@ -173,7 +173,8 @@ defmodule Pleroma.Web.WebFinger do
get_template_from_xml(body) get_template_from_xml(body)
else else
_ -> _ ->
with {:ok, %{body: body}} <- HTTP.get("https://#{domain}/.well-known/host-meta", []) do with {:ok, %{body: body, status: status}} when status in 200..299 <-
HTTP.get("https://#{domain}/.well-known/host-meta", []) do
get_template_from_xml(body) get_template_from_xml(body)
else else
e -> {:error, "Can't find LRDD template: #{inspect(e)}"} e -> {:error, "Can't find LRDD template: #{inspect(e)}"}
@ -205,7 +206,7 @@ defmodule Pleroma.Web.WebFinger do
with response <- with response <-
HTTP.get( HTTP.get(
address, address,
Accept: "application/xrd+xml,application/jrd+json" [{"accept", "application/xrd+xml,application/jrd+json"}]
), ),
{:ok, %{status: status, body: body}} when status in 200..299 <- response do {:ok, %{status: status, body: body}} when status in 200..299 <- response do
doc = XML.parse_document(body) doc = XML.parse_document(body)

10
mix.exs
View file

@ -119,7 +119,15 @@ defmodule Pleroma.Mixfile do
{:calendar, "~> 0.17.4"}, {:calendar, "~> 0.17.4"},
{:cachex, "~> 3.2"}, {:cachex, "~> 3.2"},
{:poison, "~> 3.0", override: true}, {:poison, "~> 3.0", override: true},
{:tesla, "~> 1.3", override: true}, # {:tesla, "~> 1.3", override: true},
{:tesla,
git: "https://git.pleroma.social/pleroma/elixir-libraries/tesla.git",
ref: "61b7503cef33f00834f78ddfafe0d5d9dec2270b",
override: true},
{:castore, "~> 0.1"},
{:cowlib, "~> 2.8", override: true},
{:gun,
github: "ninenines/gun", ref: "e1a69b36b180a574c0ac314ced9613fdd52312cc", override: true},
{:jason, "~> 1.0"}, {:jason, "~> 1.0"},
{:mogrify, "~> 0.6.1"}, {:mogrify, "~> 0.6.1"},
{:ex_aws, "~> 2.1"}, {:ex_aws, "~> 2.1"},

View file

@ -10,6 +10,7 @@
"cachex": {:hex, :cachex, "3.2.0", "a596476c781b0646e6cb5cd9751af2e2974c3e0d5498a8cab71807618b74fe2f", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "aef93694067a43697ae0531727e097754a9e992a1e7946296f5969d6dd9ac986"}, "cachex": {:hex, :cachex, "3.2.0", "a596476c781b0646e6cb5cd9751af2e2974c3e0d5498a8cab71807618b74fe2f", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "aef93694067a43697ae0531727e097754a9e992a1e7946296f5969d6dd9ac986"},
"calendar": {:hex, :calendar, "0.17.6", "ec291cb2e4ba499c2e8c0ef5f4ace974e2f9d02ae9e807e711a9b0c7850b9aee", [:mix], [{:tzdata, "~> 0.5.20 or ~> 0.1.201603 or ~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "738d0e17a93c2ccfe4ddc707bdc8e672e9074c8569498483feb1c4530fb91b2b"}, "calendar": {:hex, :calendar, "0.17.6", "ec291cb2e4ba499c2e8c0ef5f4ace974e2f9d02ae9e807e711a9b0c7850b9aee", [:mix], [{:tzdata, "~> 0.5.20 or ~> 0.1.201603 or ~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "738d0e17a93c2ccfe4ddc707bdc8e672e9074c8569498483feb1c4530fb91b2b"},
"captcha": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", "e0f16822d578866e186a0974d65ad58cddc1e2ab", [ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"]}, "captcha": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", "e0f16822d578866e186a0974d65ad58cddc1e2ab", [ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"]},
"castore": {:hex, :castore, "0.1.5", "591c763a637af2cc468a72f006878584bc6c306f8d111ef8ba1d4c10e0684010", [:mix], [], "hexpm", "6db356b2bc6cc22561e051ff545c20ad064af57647e436650aa24d7d06cd941a"},
"certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "805abd97539caf89ec6d4732c91e62ba9da0cda51ac462380bbd28ee697a8c42"}, "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "805abd97539caf89ec6d4732c91e62ba9da0cda51ac462380bbd28ee697a8c42"},
"combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
"comeonin": {:hex, :comeonin, "4.1.2", "3eb5620fd8e35508991664b4c2b04dd41e52f1620b36957be837c1d7784b7592", [:mix], [{:argon2_elixir, "~> 1.2", [hex: :argon2_elixir, repo: "hexpm", optional: true]}, {:bcrypt_elixir, "~> 0.12.1 or ~> 1.0", [hex: :bcrypt_elixir, repo: "hexpm", optional: true]}, {:pbkdf2_elixir, "~> 0.12", [hex: :pbkdf2_elixir, repo: "hexpm", optional: true]}], "hexpm", "d8700a0ca4dbb616c22c9b3f6dd539d88deaafec3efe66869d6370c9a559b3e9"}, "comeonin": {:hex, :comeonin, "4.1.2", "3eb5620fd8e35508991664b4c2b04dd41e52f1620b36957be837c1d7784b7592", [:mix], [{:argon2_elixir, "~> 1.2", [hex: :argon2_elixir, repo: "hexpm", optional: true]}, {:bcrypt_elixir, "~> 0.12.1 or ~> 1.0", [hex: :bcrypt_elixir, repo: "hexpm", optional: true]}, {:pbkdf2_elixir, "~> 0.12", [hex: :pbkdf2_elixir, repo: "hexpm", optional: true]}], "hexpm", "d8700a0ca4dbb616c22c9b3f6dd539d88deaafec3efe66869d6370c9a559b3e9"},
@ -46,6 +47,7 @@
"gen_stage": {:hex, :gen_stage, "0.14.3", "d0c66f1c87faa301c1a85a809a3ee9097a4264b2edf7644bf5c123237ef732bf", [:mix], [], "hexpm"}, "gen_stage": {:hex, :gen_stage, "0.14.3", "d0c66f1c87faa301c1a85a809a3ee9097a4264b2edf7644bf5c123237ef732bf", [:mix], [], "hexpm"},
"gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"}, "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"},
"gettext": {:hex, :gettext, "0.17.4", "f13088e1ec10ce01665cf25f5ff779e7df3f2dc71b37084976cf89d1aa124d5c", [:mix], [], "hexpm", "3c75b5ea8288e2ee7ea503ff9e30dfe4d07ad3c054576a6e60040e79a801e14d"}, "gettext": {:hex, :gettext, "0.17.4", "f13088e1ec10ce01665cf25f5ff779e7df3f2dc71b37084976cf89d1aa124d5c", [:mix], [], "hexpm", "3c75b5ea8288e2ee7ea503ff9e30dfe4d07ad3c054576a6e60040e79a801e14d"},
"gun": {:git, "https://github.com/ninenines/gun.git", "e1a69b36b180a574c0ac314ced9613fdd52312cc", [ref: "e1a69b36b180a574c0ac314ced9613fdd52312cc"]},
"hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "e0100f8ef7d1124222c11ad362c857d3df7cb5f4204054f9f0f4a728666591fc"}, "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "e0100f8ef7d1124222c11ad362c857d3df7cb5f4204054f9f0f4a728666591fc"},
"html_entities": {:hex, :html_entities, "0.5.1", "1c9715058b42c35a2ab65edc5b36d0ea66dd083767bef6e3edb57870ef556549", [:mix], [], "hexpm", "30efab070904eb897ff05cd52fa61c1025d7f8ef3a9ca250bc4e6513d16c32de"}, "html_entities": {:hex, :html_entities, "0.5.1", "1c9715058b42c35a2ab65edc5b36d0ea66dd083767bef6e3edb57870ef556549", [:mix], [], "hexpm", "30efab070904eb897ff05cd52fa61c1025d7f8ef3a9ca250bc4e6513d16c32de"},
"html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"},
@ -101,7 +103,7 @@
"swoosh": {:hex, :swoosh, "0.23.5", "bfd9404bbf5069b1be2ffd317923ce57e58b332e25dbca2a35dedd7820dfee5a", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "e3928e1d2889a308aaf3e42755809ac21cffd77cb58eef01cbfdab4ce2fd1e21"}, "swoosh": {:hex, :swoosh, "0.23.5", "bfd9404bbf5069b1be2ffd317923ce57e58b332e25dbca2a35dedd7820dfee5a", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "e3928e1d2889a308aaf3e42755809ac21cffd77cb58eef01cbfdab4ce2fd1e21"},
"syslog": {:hex, :syslog, "1.0.6", "995970c9aa7feb380ac493302138e308d6e04fd57da95b439a6df5bb3bf75076", [:rebar3], [], "hexpm", "769ddfabd0d2a16f3f9c17eb7509951e0ca4f68363fb26f2ee51a8ec4a49881a"}, "syslog": {:hex, :syslog, "1.0.6", "995970c9aa7feb380ac493302138e308d6e04fd57da95b439a6df5bb3bf75076", [:rebar3], [], "hexpm", "769ddfabd0d2a16f3f9c17eb7509951e0ca4f68363fb26f2ee51a8ec4a49881a"},
"telemetry": {:hex, :telemetry, "0.4.1", "ae2718484892448a24470e6aa341bc847c3277bfb8d4e9289f7474d752c09c7f", [:rebar3], [], "hexpm", "4738382e36a0a9a2b6e25d67c960e40e1a2c95560b9f936d8e29de8cd858480f"}, "telemetry": {:hex, :telemetry, "0.4.1", "ae2718484892448a24470e6aa341bc847c3277bfb8d4e9289f7474d752c09c7f", [:rebar3], [], "hexpm", "4738382e36a0a9a2b6e25d67c960e40e1a2c95560b9f936d8e29de8cd858480f"},
"tesla": {:hex, :tesla, "1.3.2", "deb92c5c9ce35e747a395ba413ca78593a4f75bf0e1545630ee2e3d34264021e", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "7567704c4790e21bd9a961b56d0b6a988ff68cc4dacfe6b2106e258da1d5cdda"}, "tesla": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/tesla.git", "61b7503cef33f00834f78ddfafe0d5d9dec2270b", [ref: "61b7503cef33f00834f78ddfafe0d5d9dec2270b"]},
"timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "f354efb2400dd7a80fd9eb6c8419068c4f632da4ac47f3d8822d6e33f08bc852"}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "f354efb2400dd7a80fd9eb6c8419068c4f632da4ac47f3d8822d6e33f08bc852"},
"trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"},
"tzdata": {:hex, :tzdata, "0.5.22", "f2ba9105117ee0360eae2eca389783ef7db36d533899b2e84559404dbc77ebb8", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "cd66c8a1e6a9e121d1f538b01bef459334bb4029a1ffb4eeeb5e4eae0337e7b6"}, "tzdata": {:hex, :tzdata, "0.5.22", "f2ba9105117ee0360eae2eca389783ef7db36d533899b2e84559404dbc77ebb8", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "cd66c8a1e6a9e121d1f538b01bef459334bb4029a1ffb4eeeb5e4eae0337e7b6"},

View file

@ -1,5 +1,5 @@
# Pleroma: A lightweight social networking server # Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/> # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Repo.Migrations.CreateConversations do defmodule Pleroma.Repo.Migrations.CreateConversations do

View file

@ -62,7 +62,7 @@ defmodule Restarter.Pleroma do
end end
def handle_cast({:restart, :test, _}, state) do def handle_cast({:restart, :test, _}, state) do
Logger.warn("pleroma restarted") Logger.debug("pleroma manually restarted")
{:noreply, Map.put(state, :need_reboot, false)} {:noreply, Map.put(state, :need_reboot, false)}
end end
@ -75,7 +75,7 @@ defmodule Restarter.Pleroma do
def handle_cast({:after_boot, _}, %{after_boot: true} = state), do: {:noreply, state} def handle_cast({:after_boot, _}, %{after_boot: true} = state), do: {:noreply, state}
def handle_cast({:after_boot, :test}, state) do def handle_cast({:after_boot, :test}, state) do
Logger.warn("pleroma restarted") Logger.debug("pleroma restarted after boot")
state = %{state | after_boot: true, rebooted: true} state = %{state | after_boot: true, rebooted: true}
{:noreply, state} {:noreply, state}
end end

View file

@ -83,7 +83,7 @@ defmodule Pleroma.Activity.Ir.TopicsTest do
assert Enum.member?(topics, "hashtag:bar") assert Enum.member?(topics, "hashtag:bar")
end end
test "only converts strinngs to hash tags", %{ test "only converts strings to hash tags", %{
activity: %{object: %{data: data} = object} = activity activity: %{object: %{data: data} = object} = activity
} do } do
tagged_data = Map.put(data, "tag", [2]) tagged_data = Map.put(data, "tag", [2])

View file

@ -478,14 +478,6 @@ defmodule Pleroma.ConfigDBTest do
assert ConfigDB.from_binary(binary) == [key: "value"] assert ConfigDB.from_binary(binary) == [key: "value"]
end end
test "keyword with partial_chain key" do
binary =
ConfigDB.transform([%{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]}])
assert binary == :erlang.term_to_binary(partial_chain: &:hackney_connect.partial_chain/1)
assert ConfigDB.from_binary(binary) == [partial_chain: &:hackney_connect.partial_chain/1]
end
test "keyword" do test "keyword" do
binary = binary =
ConfigDB.transform([ ConfigDB.transform([

41
test/fixtures/users_mock/localhost.json vendored Normal file
View file

@ -0,0 +1,41 @@
{
"@context": [
"https://www.w3.org/ns/activitystreams",
"http://localhost:4001/schemas/litepub-0.1.jsonld",
{
"@language": "und"
}
],
"attachment": [],
"endpoints": {
"oauthAuthorizationEndpoint": "http://localhost:4001/oauth/authorize",
"oauthRegistrationEndpoint": "http://localhost:4001/api/v1/apps",
"oauthTokenEndpoint": "http://localhost:4001/oauth/token",
"sharedInbox": "http://localhost:4001/inbox"
},
"followers": "http://localhost:4001/users/{{nickname}}/followers",
"following": "http://localhost:4001/users/{{nickname}}/following",
"icon": {
"type": "Image",
"url": "http://localhost:4001/media/4e914f5b84e4a259a3f6c2d2edc9ab642f2ab05f3e3d9c52c81fc2d984b3d51e.jpg"
},
"id": "http://localhost:4001/users/{{nickname}}",
"image": {
"type": "Image",
"url": "http://localhost:4001/media/f739efddefeee49c6e67e947c4811fdc911785c16ae43da4c3684051fbf8da6a.jpg?name=f739efddefeee49c6e67e947c4811fdc911785c16ae43da4c3684051fbf8da6a.jpg"
},
"inbox": "http://localhost:4001/users/{{nickname}}/inbox",
"manuallyApprovesFollowers": false,
"name": "{{nickname}}",
"outbox": "http://localhost:4001/users/{{nickname}}/outbox",
"preferredUsername": "{{nickname}}",
"publicKey": {
"id": "http://localhost:4001/users/{{nickname}}#main-key",
"owner": "http://localhost:4001/users/{{nickname}}",
"publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5DLtwGXNZElJyxFGfcVc\nXANhaMadj/iYYQwZjOJTV9QsbtiNBeIK54PJrYuU0/0YIdrvS1iqheX5IwXRhcwa\nhm3ZyLz7XeN9st7FBni4BmZMBtMpxAuYuu5p/jbWy13qAiYOhPreCx0wrWgm/lBD\n9mkgaxIxPooBE0S4ZWEJIDIV1Vft3AWcRUyWW1vIBK0uZzs6GYshbQZB952S0yo4\nFzI1hABGHncH8UvuFauh4EZ8tY7/X5I0pGRnDOcRN1dAht5w5yTA+6r5kebiFQjP\nIzN/eCO/a9Flrj9YGW7HDNtjSOH0A31PLRGlJtJO3yK57dnf5ppyCZGfL4emShQo\ncQIDAQAB\n-----END PUBLIC KEY-----\n\n"
},
"summary": "your friendly neighborhood pleroma developer<br>I like cute things and distributed systems, and really hate delete and redrafts",
"tag": [],
"type": "Person",
"url": "http://localhost:4001/users/{{nickname}}"
}

View file

@ -0,0 +1 @@
21.1

View file

@ -0,0 +1 @@
22.1

View file

@ -0,0 +1 @@
22.4

View file

@ -0,0 +1 @@
23.0

View file

@ -0,0 +1,258 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.AdapterHelper.GunTest do
use ExUnit.Case, async: true
use Pleroma.Tests.Helpers
import Mox
alias Pleroma.Config
alias Pleroma.Gun.Conn
alias Pleroma.HTTP.AdapterHelper.Gun
alias Pleroma.Pool.Connections
setup :verify_on_exit!
defp gun_mock(_) do
gun_mock()
:ok
end
defp gun_mock do
Pleroma.GunMock
|> stub(:open, fn _, _, _ -> Task.start_link(fn -> Process.sleep(1000) end) end)
|> stub(:await_up, fn _, _ -> {:ok, :http} end)
|> stub(:set_owner, fn _, _ -> :ok end)
end
describe "options/1" do
setup do: clear_config([:http, :adapter], a: 1, b: 2)
test "https url with default port" do
uri = URI.parse("https://example.com")
opts = Gun.options([receive_conn: false], uri)
assert opts[:certificates_verification]
assert opts[:tls_opts][:log_level] == :warning
end
test "https ipv4 with default port" do
uri = URI.parse("https://127.0.0.1")
opts = Gun.options([receive_conn: false], uri)
assert opts[:certificates_verification]
assert opts[:tls_opts][:log_level] == :warning
end
test "https ipv6 with default port" do
uri = URI.parse("https://[2a03:2880:f10c:83:face:b00c:0:25de]")
opts = Gun.options([receive_conn: false], uri)
assert opts[:certificates_verification]
assert opts[:tls_opts][:log_level] == :warning
end
test "https url with non standart port" do
uri = URI.parse("https://example.com:115")
opts = Gun.options([receive_conn: false], uri)
assert opts[:certificates_verification]
end
test "get conn on next request" do
gun_mock()
level = Application.get_env(:logger, :level)
Logger.configure(level: :debug)
on_exit(fn -> Logger.configure(level: level) end)
uri = URI.parse("http://some-domain2.com")
opts = Gun.options(uri)
assert opts[:conn] == nil
assert opts[:close_conn] == nil
Process.sleep(50)
opts = Gun.options(uri)
assert is_pid(opts[:conn])
assert opts[:close_conn] == false
end
test "merges with defaul http adapter config" do
defaults = Gun.options([receive_conn: false], URI.parse("https://example.com"))
assert Keyword.has_key?(defaults, :a)
assert Keyword.has_key?(defaults, :b)
end
test "default ssl adapter opts with connection" do
gun_mock()
uri = URI.parse("https://some-domain.com")
:ok = Conn.open(uri, :gun_connections)
opts = Gun.options(uri)
assert opts[:certificates_verification]
refute opts[:tls_opts] == []
assert opts[:close_conn] == false
assert is_pid(opts[:conn])
end
test "parses string proxy host & port" do
proxy = Config.get([:http, :proxy_url])
Config.put([:http, :proxy_url], "localhost:8123")
on_exit(fn -> Config.put([:http, :proxy_url], proxy) end)
uri = URI.parse("https://some-domain.com")
opts = Gun.options([receive_conn: false], uri)
assert opts[:proxy] == {'localhost', 8123}
end
test "parses tuple proxy scheme host and port" do
proxy = Config.get([:http, :proxy_url])
Config.put([:http, :proxy_url], {:socks, 'localhost', 1234})
on_exit(fn -> Config.put([:http, :proxy_url], proxy) end)
uri = URI.parse("https://some-domain.com")
opts = Gun.options([receive_conn: false], uri)
assert opts[:proxy] == {:socks, 'localhost', 1234}
end
test "passed opts have more weight than defaults" do
proxy = Config.get([:http, :proxy_url])
Config.put([:http, :proxy_url], {:socks5, 'localhost', 1234})
on_exit(fn -> Config.put([:http, :proxy_url], proxy) end)
uri = URI.parse("https://some-domain.com")
opts = Gun.options([receive_conn: false, proxy: {'example.com', 4321}], uri)
assert opts[:proxy] == {'example.com', 4321}
end
end
describe "options/1 with receive_conn parameter" do
setup :gun_mock
test "receive conn by default" do
uri = URI.parse("http://another-domain.com")
:ok = Conn.open(uri, :gun_connections)
received_opts = Gun.options(uri)
assert received_opts[:close_conn] == false
assert is_pid(received_opts[:conn])
end
test "don't receive conn if receive_conn is false" do
uri = URI.parse("http://another-domain.com")
:ok = Conn.open(uri, :gun_connections)
opts = [receive_conn: false]
received_opts = Gun.options(opts, uri)
assert received_opts[:close_conn] == nil
assert received_opts[:conn] == nil
end
end
describe "after_request/1" do
setup :gun_mock
test "body_as not chunks" do
uri = URI.parse("http://some-domain.com")
:ok = Conn.open(uri, :gun_connections)
opts = Gun.options(uri)
:ok = Gun.after_request(opts)
conn = opts[:conn]
assert %Connections{
conns: %{
"http:some-domain.com:80" => %Pleroma.Gun.Conn{
conn: ^conn,
conn_state: :idle,
used_by: []
}
}
} = Connections.get_state(:gun_connections)
end
test "body_as chunks" do
uri = URI.parse("http://some-domain.com")
:ok = Conn.open(uri, :gun_connections)
opts = Gun.options([body_as: :chunks], uri)
:ok = Gun.after_request(opts)
conn = opts[:conn]
self = self()
assert %Connections{
conns: %{
"http:some-domain.com:80" => %Pleroma.Gun.Conn{
conn: ^conn,
conn_state: :active,
used_by: [{^self, _}]
}
}
} = Connections.get_state(:gun_connections)
end
test "with no connection" do
uri = URI.parse("http://uniq-domain.com")
:ok = Conn.open(uri, :gun_connections)
opts = Gun.options([body_as: :chunks], uri)
conn = opts[:conn]
opts = Keyword.delete(opts, :conn)
self = self()
:ok = Gun.after_request(opts)
assert %Connections{
conns: %{
"http:uniq-domain.com:80" => %Pleroma.Gun.Conn{
conn: ^conn,
conn_state: :active,
used_by: [{^self, _}]
}
}
} = Connections.get_state(:gun_connections)
end
test "with ipv4" do
uri = URI.parse("http://127.0.0.1")
:ok = Conn.open(uri, :gun_connections)
opts = Gun.options(uri)
:ok = Gun.after_request(opts)
conn = opts[:conn]
assert %Connections{
conns: %{
"http:127.0.0.1:80" => %Pleroma.Gun.Conn{
conn: ^conn,
conn_state: :idle,
used_by: []
}
}
} = Connections.get_state(:gun_connections)
end
test "with ipv6" do
uri = URI.parse("http://[2a03:2880:f10c:83:face:b00c:0:25de]")
:ok = Conn.open(uri, :gun_connections)
opts = Gun.options(uri)
:ok = Gun.after_request(opts)
conn = opts[:conn]
assert %Connections{
conns: %{
"http:2a03:2880:f10c:83:face:b00c:0:25de:80" => %Pleroma.Gun.Conn{
conn: ^conn,
conn_state: :idle,
used_by: []
}
}
} = Connections.get_state(:gun_connections)
end
end
end

View file

@ -0,0 +1,47 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.AdapterHelper.HackneyTest do
use ExUnit.Case, async: true
use Pleroma.Tests.Helpers
alias Pleroma.HTTP.AdapterHelper.Hackney
setup_all do
uri = URI.parse("http://domain.com")
{:ok, uri: uri}
end
describe "options/2" do
setup do: clear_config([:http, :adapter], a: 1, b: 2)
test "add proxy and opts from config", %{uri: uri} do
opts = Hackney.options([proxy: "localhost:8123"], uri)
assert opts[:a] == 1
assert opts[:b] == 2
assert opts[:proxy] == "localhost:8123"
end
test "respect connection opts and no proxy", %{uri: uri} do
opts = Hackney.options([a: 2, b: 1], uri)
assert opts[:a] == 2
assert opts[:b] == 1
refute Keyword.has_key?(opts, :proxy)
end
test "add opts for https" do
uri = URI.parse("https://domain.com")
opts = Hackney.options(uri)
assert opts[:ssl_options] == [
partial_chain: &:hackney_connect.partial_chain/1,
versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"],
server_name_indication: 'domain.com'
]
end
end
end

View file

@ -0,0 +1,28 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.AdapterHelperTest do
use ExUnit.Case, async: true
alias Pleroma.HTTP.AdapterHelper
describe "format_proxy/1" do
test "with nil" do
assert AdapterHelper.format_proxy(nil) == nil
end
test "with string" do
assert AdapterHelper.format_proxy("127.0.0.1:8123") == {{127, 0, 0, 1}, 8123}
end
test "localhost with port" do
assert AdapterHelper.format_proxy("localhost:8123") == {'localhost', 8123}
end
test "tuple" do
assert AdapterHelper.format_proxy({:socks4, :localhost, 9050}) ==
{:socks4, 'localhost', 9050}
end
end
end

View file

@ -0,0 +1,135 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.ConnectionTest do
use ExUnit.Case, async: true
use Pleroma.Tests.Helpers
import ExUnit.CaptureLog
alias Pleroma.Config
alias Pleroma.HTTP.Connection
describe "parse_host/1" do
test "as atom to charlist" do
assert Connection.parse_host(:localhost) == 'localhost'
end
test "as string to charlist" do
assert Connection.parse_host("localhost.com") == 'localhost.com'
end
test "as string ip to tuple" do
assert Connection.parse_host("127.0.0.1") == {127, 0, 0, 1}
end
end
describe "parse_proxy/1" do
test "ip with port" do
assert Connection.parse_proxy("127.0.0.1:8123") == {:ok, {127, 0, 0, 1}, 8123}
end
test "host with port" do
assert Connection.parse_proxy("localhost:8123") == {:ok, 'localhost', 8123}
end
test "as tuple" do
assert Connection.parse_proxy({:socks4, :localhost, 9050}) ==
{:ok, :socks4, 'localhost', 9050}
end
test "as tuple with string host" do
assert Connection.parse_proxy({:socks5, "localhost", 9050}) ==
{:ok, :socks5, 'localhost', 9050}
end
end
describe "parse_proxy/1 errors" do
test "ip without port" do
capture_log(fn ->
assert Connection.parse_proxy("127.0.0.1") == {:error, :invalid_proxy}
end) =~ "parsing proxy fail \"127.0.0.1\""
end
test "host without port" do
capture_log(fn ->
assert Connection.parse_proxy("localhost") == {:error, :invalid_proxy}
end) =~ "parsing proxy fail \"localhost\""
end
test "host with bad port" do
capture_log(fn ->
assert Connection.parse_proxy("localhost:port") == {:error, :invalid_proxy_port}
end) =~ "parsing port in proxy fail \"localhost:port\""
end
test "ip with bad port" do
capture_log(fn ->
assert Connection.parse_proxy("127.0.0.1:15.9") == {:error, :invalid_proxy_port}
end) =~ "parsing port in proxy fail \"127.0.0.1:15.9\""
end
test "as tuple without port" do
capture_log(fn ->
assert Connection.parse_proxy({:socks5, :localhost}) == {:error, :invalid_proxy}
end) =~ "parsing proxy fail {:socks5, :localhost}"
end
test "with nil" do
assert Connection.parse_proxy(nil) == nil
end
end
describe "options/3" do
setup do: clear_config([:http, :proxy_url])
test "without proxy_url in config" do
Config.delete([:http, :proxy_url])
opts = Connection.options(%URI{})
refute Keyword.has_key?(opts, :proxy)
end
test "parses string proxy host & port" do
Config.put([:http, :proxy_url], "localhost:8123")
opts = Connection.options(%URI{})
assert opts[:proxy] == {'localhost', 8123}
end
test "parses tuple proxy scheme host and port" do
Config.put([:http, :proxy_url], {:socks, 'localhost', 1234})
opts = Connection.options(%URI{})
assert opts[:proxy] == {:socks, 'localhost', 1234}
end
test "passed opts have more weight than defaults" do
Config.put([:http, :proxy_url], {:socks5, 'localhost', 1234})
opts = Connection.options(%URI{}, proxy: {'example.com', 4321})
assert opts[:proxy] == {'example.com', 4321}
end
end
describe "format_host/1" do
test "with domain" do
assert Connection.format_host("example.com") == 'example.com'
end
test "with idna domain" do
assert Connection.format_host("ですexample.com") == 'xn--example-183fne.com'
end
test "with ipv4" do
assert Connection.format_host("127.0.0.1") == '127.0.0.1'
end
test "with ipv6" do
assert Connection.format_host("2a03:2880:f10c:83:face:b00c:0:25de") ==
'2a03:2880:f10c:83:face:b00c:0:25de'
end
end
end

View file

@ -5,6 +5,8 @@
defmodule Pleroma.HTTP.RequestBuilderTest do defmodule Pleroma.HTTP.RequestBuilderTest do
use ExUnit.Case, async: true use ExUnit.Case, async: true
use Pleroma.Tests.Helpers use Pleroma.Tests.Helpers
alias Pleroma.Config
alias Pleroma.HTTP.Request
alias Pleroma.HTTP.RequestBuilder alias Pleroma.HTTP.RequestBuilder
describe "headers/2" do describe "headers/2" do
@ -12,48 +14,31 @@ defmodule Pleroma.HTTP.RequestBuilderTest do
setup do: clear_config([:http, :user_agent]) setup do: clear_config([:http, :user_agent])
test "don't send pleroma user agent" do test "don't send pleroma user agent" do
assert RequestBuilder.headers(%{}, []) == %{headers: []} assert RequestBuilder.headers(%Request{}, []) == %Request{headers: []}
end end
test "send pleroma user agent" do test "send pleroma user agent" do
Pleroma.Config.put([:http, :send_user_agent], true) Config.put([:http, :send_user_agent], true)
Pleroma.Config.put([:http, :user_agent], :default) Config.put([:http, :user_agent], :default)
assert RequestBuilder.headers(%{}, []) == %{ assert RequestBuilder.headers(%Request{}, []) == %Request{
headers: [{"User-Agent", Pleroma.Application.user_agent()}] headers: [{"user-agent", Pleroma.Application.user_agent()}]
} }
end end
test "send custom user agent" do test "send custom user agent" do
Pleroma.Config.put([:http, :send_user_agent], true) Config.put([:http, :send_user_agent], true)
Pleroma.Config.put([:http, :user_agent], "totally-not-pleroma") Config.put([:http, :user_agent], "totally-not-pleroma")
assert RequestBuilder.headers(%{}, []) == %{ assert RequestBuilder.headers(%Request{}, []) == %Request{
headers: [{"User-Agent", "totally-not-pleroma"}] headers: [{"user-agent", "totally-not-pleroma"}]
} }
end end
end end
describe "add_optional_params/3" do
test "don't add if keyword is empty" do
assert RequestBuilder.add_optional_params(%{}, %{}, []) == %{}
end
test "add query parameter" do
assert RequestBuilder.add_optional_params(
%{},
%{query: :query, body: :body, another: :val},
[
{:query, "param1=val1&param2=val2"},
{:body, "some body"}
]
) == %{query: "param1=val1&param2=val2", body: "some body"}
end
end
describe "add_param/4" do describe "add_param/4" do
test "add file parameter" do test "add file parameter" do
%{ %Request{
body: %Tesla.Multipart{ body: %Tesla.Multipart{
boundary: _, boundary: _,
content_type_params: [], content_type_params: [],
@ -70,7 +55,7 @@ defmodule Pleroma.HTTP.RequestBuilderTest do
} }
] ]
} }
} = RequestBuilder.add_param(%{}, :file, "filename.png", "some-path/filename.png") } = RequestBuilder.add_param(%Request{}, :file, "filename.png", "some-path/filename.png")
end end
test "add key to body" do test "add key to body" do
@ -82,7 +67,7 @@ defmodule Pleroma.HTTP.RequestBuilderTest do
%Tesla.Multipart.Part{ %Tesla.Multipart.Part{
body: "\"someval\"", body: "\"someval\"",
dispositions: [name: "somekey"], dispositions: [name: "somekey"],
headers: ["Content-Type": "application/json"] headers: [{"content-type", "application/json"}]
} }
] ]
} }

View file

@ -3,8 +3,10 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTPTest do defmodule Pleroma.HTTPTest do
use Pleroma.DataCase use ExUnit.Case, async: true
use Pleroma.Tests.Helpers
import Tesla.Mock import Tesla.Mock
alias Pleroma.HTTP
setup do setup do
mock(fn mock(fn
@ -27,7 +29,7 @@ defmodule Pleroma.HTTPTest do
describe "get/1" do describe "get/1" do
test "returns successfully result" do test "returns successfully result" do
assert Pleroma.HTTP.get("http://example.com/hello") == { assert HTTP.get("http://example.com/hello") == {
:ok, :ok,
%Tesla.Env{status: 200, body: "hello"} %Tesla.Env{status: 200, body: "hello"}
} }
@ -36,7 +38,7 @@ defmodule Pleroma.HTTPTest do
describe "get/2 (with headers)" do describe "get/2 (with headers)" do
test "returns successfully result for json content-type" do test "returns successfully result for json content-type" do
assert Pleroma.HTTP.get("http://example.com/hello", [{"content-type", "application/json"}]) == assert HTTP.get("http://example.com/hello", [{"content-type", "application/json"}]) ==
{ {
:ok, :ok,
%Tesla.Env{ %Tesla.Env{
@ -50,7 +52,7 @@ defmodule Pleroma.HTTPTest do
describe "post/2" do describe "post/2" do
test "returns successfully result" do test "returns successfully result" do
assert Pleroma.HTTP.post("http://example.com/world", "") == { assert HTTP.post("http://example.com/world", "") == {
:ok, :ok,
%Tesla.Env{status: 200, body: "world"} %Tesla.Env{status: 200, body: "world"}
} }

View file

@ -784,12 +784,20 @@ defmodule Pleroma.NotificationTest do
"object" => remote_user.ap_id "object" => remote_user.ap_id
} }
remote_user_url = remote_user.ap_id
Tesla.Mock.mock(fn
%{method: :get, url: ^remote_user_url} ->
%Tesla.Env{status: 404, body: ""}
end)
{:ok, _delete_activity} = Transmogrifier.handle_incoming(delete_user_message) {:ok, _delete_activity} = Transmogrifier.handle_incoming(delete_user_message)
ObanHelpers.perform_all() ObanHelpers.perform_all()
assert Enum.empty?(Notification.for_user(local_user)) assert Enum.empty?(Notification.for_user(local_user))
end end
@tag capture_log: true
test "move activity generates a notification" do test "move activity generates a notification" do
%{ap_id: old_ap_id} = old_user = insert(:user) %{ap_id: old_ap_id} = old_user = insert(:user)
%{ap_id: new_ap_id} = new_user = insert(:user, also_known_as: [old_ap_id]) %{ap_id: new_ap_id} = new_user = insert(:user, also_known_as: [old_ap_id])
@ -799,6 +807,18 @@ defmodule Pleroma.NotificationTest do
User.follow(follower, old_user) User.follow(follower, old_user)
User.follow(other_follower, old_user) User.follow(other_follower, old_user)
old_user_url = old_user.ap_id
body =
File.read!("test/fixtures/users_mock/localhost.json")
|> String.replace("{{nickname}}", old_user.nickname)
|> Jason.encode!()
Tesla.Mock.mock(fn
%{method: :get, url: ^old_user_url} ->
%Tesla.Env{status: 200, body: body}
end)
Pleroma.Web.ActivityPub.ActivityPub.move(old_user, new_user) Pleroma.Web.ActivityPub.ActivityPub.move(old_user, new_user)
ObanHelpers.perform_all() ObanHelpers.perform_all()

42
test/otp_version_test.exs Normal file
View file

@ -0,0 +1,42 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.OTPVersionTest do
use ExUnit.Case, async: true
alias Pleroma.OTPVersion
describe "check/1" do
test "22.4" do
assert OTPVersion.get_version_from_files(["test/fixtures/warnings/otp_version/22.4"]) ==
"22.4"
end
test "22.1" do
assert OTPVersion.get_version_from_files(["test/fixtures/warnings/otp_version/22.1"]) ==
"22.1"
end
test "21.1" do
assert OTPVersion.get_version_from_files(["test/fixtures/warnings/otp_version/21.1"]) ==
"21.1"
end
test "23.0" do
assert OTPVersion.get_version_from_files(["test/fixtures/warnings/otp_version/23.0"]) ==
"23.0"
end
test "with non existance file" do
assert OTPVersion.get_version_from_files([
"test/fixtures/warnings/otp_version/non-exising",
"test/fixtures/warnings/otp_version/22.4"
]) == "22.4"
end
test "empty paths" do
assert OTPVersion.get_version_from_files([]) == nil
end
end
end

View file

@ -0,0 +1,760 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Pool.ConnectionsTest do
use ExUnit.Case, async: true
use Pleroma.Tests.Helpers
import ExUnit.CaptureLog
import Mox
alias Pleroma.Gun.Conn
alias Pleroma.GunMock
alias Pleroma.Pool.Connections
setup :verify_on_exit!
setup_all do
name = :test_connections
{:ok, pid} = Connections.start_link({name, [checkin_timeout: 150]})
{:ok, _} = Registry.start_link(keys: :unique, name: Pleroma.GunMock)
on_exit(fn ->
if Process.alive?(pid), do: GenServer.stop(name)
end)
{:ok, name: name}
end
defp open_mock(num \\ 1) do
GunMock
|> expect(:open, num, &start_and_register(&1, &2, &3))
|> expect(:await_up, num, fn _, _ -> {:ok, :http} end)
|> expect(:set_owner, num, fn _, _ -> :ok end)
end
defp connect_mock(mock) do
mock
|> expect(:connect, &connect(&1, &2))
|> expect(:await, &await(&1, &2))
end
defp info_mock(mock), do: expect(mock, :info, &info(&1))
defp start_and_register('gun-not-up.com', _, _), do: {:error, :timeout}
defp start_and_register(host, port, _) do
{:ok, pid} = Task.start_link(fn -> Process.sleep(1000) end)
scheme =
case port do
443 -> "https"
_ -> "http"
end
Registry.register(GunMock, pid, %{
origin_scheme: scheme,
origin_host: host,
origin_port: port
})
{:ok, pid}
end
defp info(pid) do
[{_, info}] = Registry.lookup(GunMock, pid)
info
end
defp connect(pid, _) do
ref = make_ref()
Registry.register(GunMock, ref, pid)
ref
end
defp await(pid, ref) do
[{_, ^pid}] = Registry.lookup(GunMock, ref)
{:response, :fin, 200, []}
end
defp now, do: :os.system_time(:second)
describe "alive?/2" do
test "is alive", %{name: name} do
assert Connections.alive?(name)
end
test "returns false if not started" do
refute Connections.alive?(:some_random_name)
end
end
test "opens connection and reuse it on next request", %{name: name} do
open_mock()
url = "http://some-domain.com"
key = "http:some-domain.com:80"
refute Connections.checkin(url, name)
:ok = Conn.open(url, name)
conn = Connections.checkin(url, name)
assert is_pid(conn)
assert Process.alive?(conn)
self = self()
%Connections{
conns: %{
^key => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}],
conn_state: :active
}
}
} = Connections.get_state(name)
reused_conn = Connections.checkin(url, name)
assert conn == reused_conn
%Connections{
conns: %{
^key => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}, {^self, _}],
conn_state: :active
}
}
} = Connections.get_state(name)
:ok = Connections.checkout(conn, self, name)
%Connections{
conns: %{
^key => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}],
conn_state: :active
}
}
} = Connections.get_state(name)
:ok = Connections.checkout(conn, self, name)
%Connections{
conns: %{
^key => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [],
conn_state: :idle
}
}
} = Connections.get_state(name)
end
test "reuse connection for idna domains", %{name: name} do
open_mock()
url = "http://ですsome-domain.com"
refute Connections.checkin(url, name)
:ok = Conn.open(url, name)
conn = Connections.checkin(url, name)
assert is_pid(conn)
assert Process.alive?(conn)
self = self()
%Connections{
conns: %{
"http:ですsome-domain.com:80" => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}],
conn_state: :active
}
}
} = Connections.get_state(name)
reused_conn = Connections.checkin(url, name)
assert conn == reused_conn
end
test "reuse for ipv4", %{name: name} do
open_mock()
url = "http://127.0.0.1"
refute Connections.checkin(url, name)
:ok = Conn.open(url, name)
conn = Connections.checkin(url, name)
assert is_pid(conn)
assert Process.alive?(conn)
self = self()
%Connections{
conns: %{
"http:127.0.0.1:80" => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}],
conn_state: :active
}
}
} = Connections.get_state(name)
reused_conn = Connections.checkin(url, name)
assert conn == reused_conn
:ok = Connections.checkout(conn, self, name)
:ok = Connections.checkout(reused_conn, self, name)
%Connections{
conns: %{
"http:127.0.0.1:80" => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [],
conn_state: :idle
}
}
} = Connections.get_state(name)
end
test "reuse for ipv6", %{name: name} do
open_mock()
url = "http://[2a03:2880:f10c:83:face:b00c:0:25de]"
refute Connections.checkin(url, name)
:ok = Conn.open(url, name)
conn = Connections.checkin(url, name)
assert is_pid(conn)
assert Process.alive?(conn)
self = self()
%Connections{
conns: %{
"http:2a03:2880:f10c:83:face:b00c:0:25de:80" => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}],
conn_state: :active
}
}
} = Connections.get_state(name)
reused_conn = Connections.checkin(url, name)
assert conn == reused_conn
end
test "up and down ipv4", %{name: name} do
open_mock()
|> info_mock()
|> allow(self(), name)
self = self()
url = "http://127.0.0.1"
:ok = Conn.open(url, name)
conn = Connections.checkin(url, name)
send(name, {:gun_down, conn, nil, nil, nil})
send(name, {:gun_up, conn, nil})
%Connections{
conns: %{
"http:127.0.0.1:80" => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}],
conn_state: :active
}
}
} = Connections.get_state(name)
end
test "up and down ipv6", %{name: name} do
self = self()
open_mock()
|> info_mock()
|> allow(self, name)
url = "http://[2a03:2880:f10c:83:face:b00c:0:25de]"
:ok = Conn.open(url, name)
conn = Connections.checkin(url, name)
send(name, {:gun_down, conn, nil, nil, nil})
send(name, {:gun_up, conn, nil})
%Connections{
conns: %{
"http:2a03:2880:f10c:83:face:b00c:0:25de:80" => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}],
conn_state: :active
}
}
} = Connections.get_state(name)
end
test "reuses connection based on protocol", %{name: name} do
open_mock(2)
http_url = "http://some-domain.com"
http_key = "http:some-domain.com:80"
https_url = "https://some-domain.com"
https_key = "https:some-domain.com:443"
refute Connections.checkin(http_url, name)
:ok = Conn.open(http_url, name)
conn = Connections.checkin(http_url, name)
assert is_pid(conn)
assert Process.alive?(conn)
refute Connections.checkin(https_url, name)
:ok = Conn.open(https_url, name)
https_conn = Connections.checkin(https_url, name)
refute conn == https_conn
reused_https = Connections.checkin(https_url, name)
refute conn == reused_https
assert reused_https == https_conn
%Connections{
conns: %{
^http_key => %Conn{
conn: ^conn,
gun_state: :up
},
^https_key => %Conn{
conn: ^https_conn,
gun_state: :up
}
}
} = Connections.get_state(name)
end
test "connection can't get up", %{name: name} do
expect(GunMock, :open, &start_and_register(&1, &2, &3))
url = "http://gun-not-up.com"
assert capture_log(fn ->
refute Conn.open(url, name)
refute Connections.checkin(url, name)
end) =~
"Opening connection to http://gun-not-up.com failed with error {:error, :timeout}"
end
test "process gun_down message and then gun_up", %{name: name} do
self = self()
open_mock()
|> info_mock()
|> allow(self, name)
url = "http://gun-down-and-up.com"
key = "http:gun-down-and-up.com:80"
:ok = Conn.open(url, name)
conn = Connections.checkin(url, name)
assert is_pid(conn)
assert Process.alive?(conn)
%Connections{
conns: %{
^key => %Conn{
conn: ^conn,
gun_state: :up,
used_by: [{^self, _}]
}
}
} = Connections.get_state(name)
send(name, {:gun_down, conn, :http, nil, nil})
%Connections{
conns: %{
^key => %Conn{
conn: ^conn,
gun_state: :down,
used_by: [{^self, _}]
}
}
} = Connections.get_state(name)
send(name, {:gun_up, conn, :http})
conn2 = Connections.checkin(url, name)
assert conn == conn2
assert is_pid(conn2)
assert Process.alive?(conn2)
%Connections{
conns: %{
^key => %Conn{
conn: _,
gun_state: :up,
used_by: [{^self, _}, {^self, _}]
}
}
} = Connections.get_state(name)
end
test "async processes get same conn for same domain", %{name: name} do
open_mock()
url = "http://some-domain.com"
:ok = Conn.open(url, name)
tasks =
for _ <- 1..5 do
Task.async(fn ->
Connections.checkin(url, name)
end)
end
tasks_with_results = Task.yield_many(tasks)
results =
Enum.map(tasks_with_results, fn {task, res} ->
res || Task.shutdown(task, :brutal_kill)
end)
conns = for {:ok, value} <- results, do: value
%Connections{
conns: %{
"http:some-domain.com:80" => %Conn{
conn: conn,
gun_state: :up
}
}
} = Connections.get_state(name)
assert Enum.all?(conns, fn res -> res == conn end)
end
test "remove frequently used and idle", %{name: name} do
open_mock(3)
self = self()
http_url = "http://some-domain.com"
https_url = "https://some-domain.com"
:ok = Conn.open(https_url, name)
:ok = Conn.open(http_url, name)
conn1 = Connections.checkin(https_url, name)
[conn2 | _conns] =
for _ <- 1..4 do
Connections.checkin(http_url, name)
end
http_key = "http:some-domain.com:80"
%Connections{
conns: %{
^http_key => %Conn{
conn: ^conn2,
gun_state: :up,
conn_state: :active,
used_by: [{^self, _}, {^self, _}, {^self, _}, {^self, _}]
},
"https:some-domain.com:443" => %Conn{
conn: ^conn1,
gun_state: :up,
conn_state: :active,
used_by: [{^self, _}]
}
}
} = Connections.get_state(name)
:ok = Connections.checkout(conn1, self, name)
another_url = "http://another-domain.com"
:ok = Conn.open(another_url, name)
conn = Connections.checkin(another_url, name)
%Connections{
conns: %{
"http:another-domain.com:80" => %Conn{
conn: ^conn,
gun_state: :up
},
^http_key => %Conn{
conn: _,
gun_state: :up
}
}
} = Connections.get_state(name)
end
describe "with proxy" do
test "as ip", %{name: name} do
open_mock()
|> connect_mock()
url = "http://proxy-string.com"
key = "http:proxy-string.com:80"
:ok = Conn.open(url, name, proxy: {{127, 0, 0, 1}, 8123})
conn = Connections.checkin(url, name)
%Connections{
conns: %{
^key => %Conn{
conn: ^conn,
gun_state: :up
}
}
} = Connections.get_state(name)
reused_conn = Connections.checkin(url, name)
assert reused_conn == conn
end
test "as host", %{name: name} do
open_mock()
|> connect_mock()
url = "http://proxy-tuple-atom.com"
:ok = Conn.open(url, name, proxy: {'localhost', 9050})
conn = Connections.checkin(url, name)
%Connections{
conns: %{
"http:proxy-tuple-atom.com:80" => %Conn{
conn: ^conn,
gun_state: :up
}
}
} = Connections.get_state(name)
reused_conn = Connections.checkin(url, name)
assert reused_conn == conn
end
test "as ip and ssl", %{name: name} do
open_mock()
|> connect_mock()
url = "https://proxy-string.com"
:ok = Conn.open(url, name, proxy: {{127, 0, 0, 1}, 8123})
conn = Connections.checkin(url, name)
%Connections{
conns: %{
"https:proxy-string.com:443" => %Conn{
conn: ^conn,
gun_state: :up
}
}
} = Connections.get_state(name)
reused_conn = Connections.checkin(url, name)
assert reused_conn == conn
end
test "as host and ssl", %{name: name} do
open_mock()
|> connect_mock()
url = "https://proxy-tuple-atom.com"
:ok = Conn.open(url, name, proxy: {'localhost', 9050})
conn = Connections.checkin(url, name)
%Connections{
conns: %{
"https:proxy-tuple-atom.com:443" => %Conn{
conn: ^conn,
gun_state: :up
}
}
} = Connections.get_state(name)
reused_conn = Connections.checkin(url, name)
assert reused_conn == conn
end
test "with socks type", %{name: name} do
open_mock()
url = "http://proxy-socks.com"
:ok = Conn.open(url, name, proxy: {:socks5, 'localhost', 1234})
conn = Connections.checkin(url, name)
%Connections{
conns: %{
"http:proxy-socks.com:80" => %Conn{
conn: ^conn,
gun_state: :up
}
}
} = Connections.get_state(name)
reused_conn = Connections.checkin(url, name)
assert reused_conn == conn
end
test "with socks4 type and ssl", %{name: name} do
open_mock()
url = "https://proxy-socks.com"
:ok = Conn.open(url, name, proxy: {:socks4, 'localhost', 1234})
conn = Connections.checkin(url, name)
%Connections{
conns: %{
"https:proxy-socks.com:443" => %Conn{
conn: ^conn,
gun_state: :up
}
}
} = Connections.get_state(name)
reused_conn = Connections.checkin(url, name)
assert reused_conn == conn
end
end
describe "crf/3" do
setup do
crf = Connections.crf(1, 10, 1)
{:ok, crf: crf}
end
test "more used will have crf higher", %{crf: crf} do
# used 3 times
crf1 = Connections.crf(1, 10, crf)
crf1 = Connections.crf(1, 10, crf1)
# used 2 times
crf2 = Connections.crf(1, 10, crf)
assert crf1 > crf2
end
test "recently used will have crf higher on equal references", %{crf: crf} do
# used 3 sec ago
crf1 = Connections.crf(3, 10, crf)
# used 4 sec ago
crf2 = Connections.crf(4, 10, crf)
assert crf1 > crf2
end
test "equal crf on equal reference and time", %{crf: crf} do
# used 2 times
crf1 = Connections.crf(1, 10, crf)
# used 2 times
crf2 = Connections.crf(1, 10, crf)
assert crf1 == crf2
end
test "recently used will have higher crf", %{crf: crf} do
crf1 = Connections.crf(2, 10, crf)
crf1 = Connections.crf(1, 10, crf1)
crf2 = Connections.crf(3, 10, crf)
crf2 = Connections.crf(4, 10, crf2)
assert crf1 > crf2
end
end
describe "get_unused_conns/1" do
test "crf is equalent, sorting by reference", %{name: name} do
Connections.add_conn(name, "1", %Conn{
conn_state: :idle,
last_reference: now() - 1
})
Connections.add_conn(name, "2", %Conn{
conn_state: :idle,
last_reference: now()
})
assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(name)
end
test "reference is equalent, sorting by crf", %{name: name} do
Connections.add_conn(name, "1", %Conn{
conn_state: :idle,
crf: 1.999
})
Connections.add_conn(name, "2", %Conn{
conn_state: :idle,
crf: 2
})
assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(name)
end
test "higher crf and lower reference", %{name: name} do
Connections.add_conn(name, "1", %Conn{
conn_state: :idle,
crf: 3,
last_reference: now() - 1
})
Connections.add_conn(name, "2", %Conn{
conn_state: :idle,
crf: 2,
last_reference: now()
})
assert [{"2", _unused_conn} | _others] = Connections.get_unused_conns(name)
end
test "lower crf and lower reference", %{name: name} do
Connections.add_conn(name, "1", %Conn{
conn_state: :idle,
crf: 1.99,
last_reference: now() - 1
})
Connections.add_conn(name, "2", %Conn{
conn_state: :idle,
crf: 2,
last_reference: now()
})
assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(name)
end
end
test "count/1" do
name = :test_count
{:ok, _} = Connections.start_link({name, [checkin_timeout: 150]})
assert Connections.count(name) == 0
Connections.add_conn(name, "1", %Conn{conn: self()})
assert Connections.count(name) == 1
Connections.remove_conn(name, "1")
assert Connections.count(name) == 0
end
end

View file

@ -4,13 +4,16 @@
defmodule Pleroma.ReverseProxyTest do defmodule Pleroma.ReverseProxyTest do
use Pleroma.Web.ConnCase, async: true use Pleroma.Web.ConnCase, async: true
import ExUnit.CaptureLog import ExUnit.CaptureLog
import Mox import Mox
alias Pleroma.ReverseProxy alias Pleroma.ReverseProxy
alias Pleroma.ReverseProxy.ClientMock alias Pleroma.ReverseProxy.ClientMock
alias Plug.Conn
setup_all do setup_all do
{:ok, _} = Registry.start_link(keys: :unique, name: Pleroma.ReverseProxy.ClientMock) {:ok, _} = Registry.start_link(keys: :unique, name: ClientMock)
:ok :ok
end end
@ -21,7 +24,7 @@ defmodule Pleroma.ReverseProxyTest do
ClientMock ClientMock
|> expect(:request, fn :get, url, _, _, _ -> |> expect(:request, fn :get, url, _, _, _ ->
Registry.register(Pleroma.ReverseProxy.ClientMock, url, 0) Registry.register(ClientMock, url, 0)
{:ok, 200, {:ok, 200,
[ [
@ -29,14 +32,14 @@ defmodule Pleroma.ReverseProxyTest do
{"content-length", byte_size(json) |> to_string()} {"content-length", byte_size(json) |> to_string()}
], %{url: url}} ], %{url: url}}
end) end)
|> expect(:stream_body, invokes, fn %{url: url} -> |> expect(:stream_body, invokes, fn %{url: url} = client ->
case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do case Registry.lookup(ClientMock, url) do
[{_, 0}] -> [{_, 0}] ->
Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1)) Registry.update_value(ClientMock, url, &(&1 + 1))
{:ok, json} {:ok, json, client}
[{_, 1}] -> [{_, 1}] ->
Registry.unregister(Pleroma.ReverseProxy.ClientMock, url) Registry.unregister(ClientMock, url)
:done :done
end end
end) end)
@ -78,7 +81,39 @@ defmodule Pleroma.ReverseProxyTest do
assert conn.halted assert conn.halted
end end
describe "max_body " do defp stream_mock(invokes, with_close? \\ false) do
ClientMock
|> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ ->
Registry.register(ClientMock, "/stream-bytes/" <> length, 0)
{:ok, 200, [{"content-type", "application/octet-stream"}],
%{url: "/stream-bytes/" <> length}}
end)
|> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} = client ->
max = String.to_integer(length)
case Registry.lookup(ClientMock, "/stream-bytes/" <> length) do
[{_, current}] when current < max ->
Registry.update_value(
ClientMock,
"/stream-bytes/" <> length,
&(&1 + 10)
)
{:ok, "0123456789", client}
[{_, ^max}] ->
Registry.unregister(ClientMock, "/stream-bytes/" <> length)
:done
end
end)
if with_close? do
expect(ClientMock, :close, fn _ -> :ok end)
end
end
describe "max_body" do
test "length returns error if content-length more than option", %{conn: conn} do test "length returns error if content-length more than option", %{conn: conn} do
user_agent_mock("hackney/1.15.1", 0) user_agent_mock("hackney/1.15.1", 0)
@ -94,38 +129,6 @@ defmodule Pleroma.ReverseProxyTest do
end) == "" end) == ""
end end
defp stream_mock(invokes, with_close? \\ false) do
ClientMock
|> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ ->
Registry.register(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length, 0)
{:ok, 200, [{"content-type", "application/octet-stream"}],
%{url: "/stream-bytes/" <> length}}
end)
|> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} ->
max = String.to_integer(length)
case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) do
[{_, current}] when current < max ->
Registry.update_value(
Pleroma.ReverseProxy.ClientMock,
"/stream-bytes/" <> length,
&(&1 + 10)
)
{:ok, "0123456789"}
[{_, ^max}] ->
Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length)
:done
end
end)
if with_close? do
expect(ClientMock, :close, fn _ -> :ok end)
end
end
test "max_body_length returns error if streaming body more than that option", %{conn: conn} do test "max_body_length returns error if streaming body more than that option", %{conn: conn} do
stream_mock(3, true) stream_mock(3, true)
@ -214,24 +217,24 @@ defmodule Pleroma.ReverseProxyTest do
conn = ReverseProxy.call(conn, "/stream-bytes/200") conn = ReverseProxy.call(conn, "/stream-bytes/200")
assert conn.state == :chunked assert conn.state == :chunked
assert byte_size(conn.resp_body) == 200 assert byte_size(conn.resp_body) == 200
assert Plug.Conn.get_resp_header(conn, "content-type") == ["application/octet-stream"] assert Conn.get_resp_header(conn, "content-type") == ["application/octet-stream"]
end end
defp headers_mock(_) do defp headers_mock(_) do
ClientMock ClientMock
|> expect(:request, fn :get, "/headers", headers, _, _ -> |> expect(:request, fn :get, "/headers", headers, _, _ ->
Registry.register(Pleroma.ReverseProxy.ClientMock, "/headers", 0) Registry.register(ClientMock, "/headers", 0)
{:ok, 200, [{"content-type", "application/json"}], %{url: "/headers", headers: headers}} {:ok, 200, [{"content-type", "application/json"}], %{url: "/headers", headers: headers}}
end) end)
|> expect(:stream_body, 2, fn %{url: url, headers: headers} -> |> expect(:stream_body, 2, fn %{url: url, headers: headers} = client ->
case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do case Registry.lookup(ClientMock, url) do
[{_, 0}] -> [{_, 0}] ->
Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1)) Registry.update_value(ClientMock, url, &(&1 + 1))
headers = for {k, v} <- headers, into: %{}, do: {String.capitalize(k), v} headers = for {k, v} <- headers, into: %{}, do: {String.capitalize(k), v}
{:ok, Jason.encode!(%{headers: headers})} {:ok, Jason.encode!(%{headers: headers}), client}
[{_, 1}] -> [{_, 1}] ->
Registry.unregister(Pleroma.ReverseProxy.ClientMock, url) Registry.unregister(ClientMock, url)
:done :done
end end
end) end)
@ -244,7 +247,7 @@ defmodule Pleroma.ReverseProxyTest do
test "header passes", %{conn: conn} do test "header passes", %{conn: conn} do
conn = conn =
Plug.Conn.put_req_header( Conn.put_req_header(
conn, conn,
"accept", "accept",
"text/html" "text/html"
@ -257,7 +260,7 @@ defmodule Pleroma.ReverseProxyTest do
test "header is filtered", %{conn: conn} do test "header is filtered", %{conn: conn} do
conn = conn =
Plug.Conn.put_req_header( Conn.put_req_header(
conn, conn,
"accept-language", "accept-language",
"en-US" "en-US"
@ -290,18 +293,18 @@ defmodule Pleroma.ReverseProxyTest do
defp disposition_headers_mock(headers) do defp disposition_headers_mock(headers) do
ClientMock ClientMock
|> expect(:request, fn :get, "/disposition", _, _, _ -> |> expect(:request, fn :get, "/disposition", _, _, _ ->
Registry.register(Pleroma.ReverseProxy.ClientMock, "/disposition", 0) Registry.register(ClientMock, "/disposition", 0)
{:ok, 200, headers, %{url: "/disposition"}} {:ok, 200, headers, %{url: "/disposition"}}
end) end)
|> expect(:stream_body, 2, fn %{url: "/disposition"} -> |> expect(:stream_body, 2, fn %{url: "/disposition"} = client ->
case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/disposition") do case Registry.lookup(ClientMock, "/disposition") do
[{_, 0}] -> [{_, 0}] ->
Registry.update_value(Pleroma.ReverseProxy.ClientMock, "/disposition", &(&1 + 1)) Registry.update_value(ClientMock, "/disposition", &(&1 + 1))
{:ok, ""} {:ok, "", client}
[{_, 1}] -> [{_, 1}] ->
Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/disposition") Registry.unregister(ClientMock, "/disposition")
:done :done
end end
end) end)

View file

@ -107,7 +107,7 @@ defmodule HttpRequestMock do
"https://osada.macgirvin.com/.well-known/webfinger?resource=acct:mike@osada.macgirvin.com", "https://osada.macgirvin.com/.well-known/webfinger?resource=acct:mike@osada.macgirvin.com",
_, _,
_, _,
Accept: "application/xrd+xml,application/jrd+json" [{"accept", "application/xrd+xml,application/jrd+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -120,7 +120,7 @@ defmodule HttpRequestMock do
"https://social.heldscal.la/.well-known/webfinger?resource=https://social.heldscal.la/user/29191", "https://social.heldscal.la/.well-known/webfinger?resource=https://social.heldscal.la/user/29191",
_, _,
_, _,
Accept: "application/xrd+xml,application/jrd+json" [{"accept", "application/xrd+xml,application/jrd+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -141,7 +141,7 @@ defmodule HttpRequestMock do
"https://pawoo.net/.well-known/webfinger?resource=acct:https://pawoo.net/users/pekorino", "https://pawoo.net/.well-known/webfinger?resource=acct:https://pawoo.net/users/pekorino",
_, _,
_, _,
Accept: "application/xrd+xml,application/jrd+json" [{"accept", "application/xrd+xml,application/jrd+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -167,7 +167,7 @@ defmodule HttpRequestMock do
"https://social.stopwatchingus-heidelberg.de/.well-known/webfinger?resource=acct:https://social.stopwatchingus-heidelberg.de/user/18330", "https://social.stopwatchingus-heidelberg.de/.well-known/webfinger?resource=acct:https://social.stopwatchingus-heidelberg.de/user/18330",
_, _,
_, _,
Accept: "application/xrd+xml,application/jrd+json" [{"accept", "application/xrd+xml,application/jrd+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -188,7 +188,7 @@ defmodule HttpRequestMock do
"https://mamot.fr/.well-known/webfinger?resource=acct:https://mamot.fr/users/Skruyb", "https://mamot.fr/.well-known/webfinger?resource=acct:https://mamot.fr/users/Skruyb",
_, _,
_, _,
Accept: "application/xrd+xml,application/jrd+json" [{"accept", "application/xrd+xml,application/jrd+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -201,7 +201,7 @@ defmodule HttpRequestMock do
"https://social.heldscal.la/.well-known/webfinger?resource=nonexistant@social.heldscal.la", "https://social.heldscal.la/.well-known/webfinger?resource=nonexistant@social.heldscal.la",
_, _,
_, _,
Accept: "application/xrd+xml,application/jrd+json" [{"accept", "application/xrd+xml,application/jrd+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -214,7 +214,7 @@ defmodule HttpRequestMock do
"https://squeet.me/xrd/?uri=lain@squeet.me", "https://squeet.me/xrd/?uri=lain@squeet.me",
_, _,
_, _,
Accept: "application/xrd+xml,application/jrd+json" [{"accept", "application/xrd+xml,application/jrd+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -227,7 +227,7 @@ defmodule HttpRequestMock do
"https://mst3k.interlinked.me/users/luciferMysticus", "https://mst3k.interlinked.me/users/luciferMysticus",
_, _,
_, _,
Accept: "application/activity+json" [{"accept", "application/activity+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -248,7 +248,7 @@ defmodule HttpRequestMock do
"https://hubzilla.example.org/channel/kaniini", "https://hubzilla.example.org/channel/kaniini",
_, _,
_, _,
Accept: "application/activity+json" [{"accept", "application/activity+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -257,7 +257,7 @@ defmodule HttpRequestMock do
}} }}
end end
def get("https://niu.moe/users/rye", _, _, Accept: "application/activity+json") do def get("https://niu.moe/users/rye", _, _, [{"accept", "application/activity+json"}]) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
status: 200, status: 200,
@ -265,7 +265,7 @@ defmodule HttpRequestMock do
}} }}
end end
def get("https://n1u.moe/users/rye", _, _, Accept: "application/activity+json") do def get("https://n1u.moe/users/rye", _, _, [{"accept", "application/activity+json"}]) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
status: 200, status: 200,
@ -284,7 +284,7 @@ defmodule HttpRequestMock do
}} }}
end end
def get("https://puckipedia.com/", _, _, Accept: "application/activity+json") do def get("https://puckipedia.com/", _, _, [{"accept", "application/activity+json"}]) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
status: 200, status: 200,
@ -308,9 +308,9 @@ defmodule HttpRequestMock do
}} }}
end end
def get("https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39", _, _, def get("https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39", _, _, [
Accept: "application/activity+json" {"accept", "application/activity+json"}
) do ]) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
status: 200, status: 200,
@ -318,7 +318,7 @@ defmodule HttpRequestMock do
}} }}
end end
def get("https://mobilizon.org/@tcit", _, _, Accept: "application/activity+json") do def get("https://mobilizon.org/@tcit", _, _, [{"accept", "application/activity+json"}]) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
status: 200, status: 200,
@ -358,7 +358,7 @@ defmodule HttpRequestMock do
}} }}
end end
def get("http://mastodon.example.org/users/admin", _, _, Accept: "application/activity+json") do def get("http://mastodon.example.org/users/admin", _, _, _) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
status: 200, status: 200,
@ -366,7 +366,9 @@ defmodule HttpRequestMock do
}} }}
end end
def get("http://mastodon.example.org/users/relay", _, _, Accept: "application/activity+json") do def get("http://mastodon.example.org/users/relay", _, _, [
{"accept", "application/activity+json"}
]) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
status: 200, status: 200,
@ -374,7 +376,9 @@ defmodule HttpRequestMock do
}} }}
end end
def get("http://mastodon.example.org/users/gargron", _, _, Accept: "application/activity+json") do def get("http://mastodon.example.org/users/gargron", _, _, [
{"accept", "application/activity+json"}
]) do
{:error, :nxdomain} {:error, :nxdomain}
end end
@ -557,7 +561,7 @@ defmodule HttpRequestMock do
"http://mastodon.example.org/@admin/99541947525187367", "http://mastodon.example.org/@admin/99541947525187367",
_, _,
_, _,
Accept: "application/activity+json" _
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -582,7 +586,7 @@ defmodule HttpRequestMock do
}} }}
end end
def get("https://mstdn.io/users/mayuutann", _, _, Accept: "application/activity+json") do def get("https://mstdn.io/users/mayuutann", _, _, [{"accept", "application/activity+json"}]) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
status: 200, status: 200,
@ -594,7 +598,7 @@ defmodule HttpRequestMock do
"https://mstdn.io/users/mayuutann/statuses/99568293732299394", "https://mstdn.io/users/mayuutann/statuses/99568293732299394",
_, _,
_, _,
Accept: "application/activity+json" [{"accept", "application/activity+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -614,7 +618,7 @@ defmodule HttpRequestMock do
}} }}
end end
def get(url, _, _, Accept: "application/xrd+xml,application/jrd+json") def get(url, _, _, [{"accept", "application/xrd+xml,application/jrd+json"}])
when url in [ when url in [
"https://pleroma.soykaf.com/.well-known/webfinger?resource=acct:https://pleroma.soykaf.com/users/lain", "https://pleroma.soykaf.com/.well-known/webfinger?resource=acct:https://pleroma.soykaf.com/users/lain",
"https://pleroma.soykaf.com/.well-known/webfinger?resource=https://pleroma.soykaf.com/users/lain" "https://pleroma.soykaf.com/.well-known/webfinger?resource=https://pleroma.soykaf.com/users/lain"
@ -641,7 +645,7 @@ defmodule HttpRequestMock do
"https://shitposter.club/.well-known/webfinger?resource=https://shitposter.club/user/1", "https://shitposter.club/.well-known/webfinger?resource=https://shitposter.club/user/1",
_, _,
_, _,
Accept: "application/xrd+xml,application/jrd+json" [{"accept", "application/xrd+xml,application/jrd+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -685,7 +689,7 @@ defmodule HttpRequestMock do
"https://shitposter.club/.well-known/webfinger?resource=https://shitposter.club/user/5381", "https://shitposter.club/.well-known/webfinger?resource=https://shitposter.club/user/5381",
_, _,
_, _,
Accept: "application/xrd+xml,application/jrd+json" [{"accept", "application/xrd+xml,application/jrd+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -738,7 +742,7 @@ defmodule HttpRequestMock do
"https://social.sakamoto.gq/.well-known/webfinger?resource=https://social.sakamoto.gq/users/eal", "https://social.sakamoto.gq/.well-known/webfinger?resource=https://social.sakamoto.gq/users/eal",
_, _,
_, _,
Accept: "application/xrd+xml,application/jrd+json" [{"accept", "application/xrd+xml,application/jrd+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -751,7 +755,7 @@ defmodule HttpRequestMock do
"https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056", "https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056",
_, _,
_, _,
Accept: "application/atom+xml" [{"accept", "application/atom+xml"}]
) do ) do
{:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/sakamoto.atom")}} {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/sakamoto.atom")}}
end end
@ -768,7 +772,7 @@ defmodule HttpRequestMock do
"https://mastodon.social/.well-known/webfinger?resource=https://mastodon.social/users/lambadalambda", "https://mastodon.social/.well-known/webfinger?resource=https://mastodon.social/users/lambadalambda",
_, _,
_, _,
Accept: "application/xrd+xml,application/jrd+json" [{"accept", "application/xrd+xml,application/jrd+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -790,7 +794,7 @@ defmodule HttpRequestMock do
"http://gs.example.org/.well-known/webfinger?resource=http://gs.example.org:4040/index.php/user/1", "http://gs.example.org/.well-known/webfinger?resource=http://gs.example.org:4040/index.php/user/1",
_, _,
_, _,
Accept: "application/xrd+xml,application/jrd+json" [{"accept", "application/xrd+xml,application/jrd+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -804,7 +808,7 @@ defmodule HttpRequestMock do
"http://gs.example.org:4040/index.php/user/1", "http://gs.example.org:4040/index.php/user/1",
_, _,
_, _,
Accept: "application/activity+json" [{"accept", "application/activity+json"}]
) do ) do
{:ok, %Tesla.Env{status: 406, body: ""}} {:ok, %Tesla.Env{status: 406, body: ""}}
end end
@ -840,7 +844,7 @@ defmodule HttpRequestMock do
"https://squeet.me/xrd?uri=lain@squeet.me", "https://squeet.me/xrd?uri=lain@squeet.me",
_, _,
_, _,
Accept: "application/xrd+xml,application/jrd+json" [{"accept", "application/xrd+xml,application/jrd+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -853,7 +857,7 @@ defmodule HttpRequestMock do
"https://social.heldscal.la/.well-known/webfinger?resource=shp@social.heldscal.la", "https://social.heldscal.la/.well-known/webfinger?resource=shp@social.heldscal.la",
_, _,
_, _,
Accept: "application/xrd+xml,application/jrd+json" [{"accept", "application/xrd+xml,application/jrd+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -866,7 +870,7 @@ defmodule HttpRequestMock do
"https://social.heldscal.la/.well-known/webfinger?resource=invalid_content@social.heldscal.la", "https://social.heldscal.la/.well-known/webfinger?resource=invalid_content@social.heldscal.la",
_, _,
_, _,
Accept: "application/xrd+xml,application/jrd+json" [{"accept", "application/xrd+xml,application/jrd+json"}]
) do ) do
{:ok, %Tesla.Env{status: 200, body: ""}} {:ok, %Tesla.Env{status: 200, body: ""}}
end end
@ -883,7 +887,7 @@ defmodule HttpRequestMock do
"http://framatube.org/main/xrd?uri=framasoft@framatube.org", "http://framatube.org/main/xrd?uri=framasoft@framatube.org",
_, _,
_, _,
Accept: "application/xrd+xml,application/jrd+json" [{"accept", "application/xrd+xml,application/jrd+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -905,7 +909,7 @@ defmodule HttpRequestMock do
"http://gnusocial.de/main/xrd?uri=winterdienst@gnusocial.de", "http://gnusocial.de/main/xrd?uri=winterdienst@gnusocial.de",
_, _,
_, _,
Accept: "application/xrd+xml,application/jrd+json" [{"accept", "application/xrd+xml,application/jrd+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -942,7 +946,7 @@ defmodule HttpRequestMock do
"https://gerzilla.de/xrd/?uri=kaniini@gerzilla.de", "https://gerzilla.de/xrd/?uri=kaniini@gerzilla.de",
_, _,
_, _,
Accept: "application/xrd+xml,application/jrd+json" [{"accept", "application/xrd+xml,application/jrd+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -1005,7 +1009,7 @@ defmodule HttpRequestMock do
%Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/osada-user-indio.json")}} %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/osada-user-indio.json")}}
end end
def get("https://social.heldscal.la/user/23211", _, _, Accept: "application/activity+json") do def get("https://social.heldscal.la/user/23211", _, _, [{"accept", "application/activity+json"}]) do
{:ok, Tesla.Mock.json(%{"id" => "https://social.heldscal.la/user/23211"}, status: 200)} {:ok, Tesla.Mock.json(%{"id" => "https://social.heldscal.la/user/23211"}, status: 200)}
end end
@ -1138,7 +1142,7 @@ defmodule HttpRequestMock do
"https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=lain@zetsubou.xn--q9jyb4c", "https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=lain@zetsubou.xn--q9jyb4c",
_, _,
_, _,
Accept: "application/xrd+xml,application/jrd+json" [{"accept", "application/xrd+xml,application/jrd+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -1151,7 +1155,7 @@ defmodule HttpRequestMock do
"https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=https://zetsubou.xn--q9jyb4c/users/lain", "https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=https://zetsubou.xn--q9jyb4c/users/lain",
_, _,
_, _,
Accept: "application/xrd+xml,application/jrd+json" [{"accept", "application/xrd+xml,application/jrd+json"}]
) do ) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
@ -1173,7 +1177,9 @@ defmodule HttpRequestMock do
}} }}
end end
def get("https://info.pleroma.site/activity.json", _, _, Accept: "application/activity+json") do def get("https://info.pleroma.site/activity.json", _, _, [
{"accept", "application/activity+json"}
]) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
status: 200, status: 200,
@ -1185,7 +1191,9 @@ defmodule HttpRequestMock do
{:ok, %Tesla.Env{status: 404, body: ""}} {:ok, %Tesla.Env{status: 404, body: ""}}
end end
def get("https://info.pleroma.site/activity2.json", _, _, Accept: "application/activity+json") do def get("https://info.pleroma.site/activity2.json", _, _, [
{"accept", "application/activity+json"}
]) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
status: 200, status: 200,
@ -1197,7 +1205,9 @@ defmodule HttpRequestMock do
{:ok, %Tesla.Env{status: 404, body: ""}} {:ok, %Tesla.Env{status: 404, body: ""}}
end end
def get("https://info.pleroma.site/activity3.json", _, _, Accept: "application/activity+json") do def get("https://info.pleroma.site/activity3.json", _, _, [
{"accept", "application/activity+json"}
]) do
{:ok, {:ok,
%Tesla.Env{ %Tesla.Env{
status: 200, status: 200,

View file

@ -6,7 +6,10 @@ os_exclude = if :os.type() == {:unix, :darwin}, do: [skip_on_mac: true], else: [
ExUnit.start(exclude: [:federated | os_exclude]) ExUnit.start(exclude: [:federated | os_exclude])
Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, :manual) Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, :manual)
Mox.defmock(Pleroma.ReverseProxy.ClientMock, for: Pleroma.ReverseProxy.Client) Mox.defmock(Pleroma.ReverseProxy.ClientMock, for: Pleroma.ReverseProxy.Client)
Mox.defmock(Pleroma.GunMock, for: Pleroma.Gun)
{:ok, _} = Application.ensure_all_started(:ex_machina) {:ok, _} = Application.ensure_all_started(:ex_machina)
ExUnit.after_suite(fn _results -> ExUnit.after_suite(fn _results ->

View file

@ -4,7 +4,6 @@
defmodule Pleroma.UserInviteTokenTest do defmodule Pleroma.UserInviteTokenTest do
use ExUnit.Case, async: true use ExUnit.Case, async: true
use Pleroma.DataCase
alias Pleroma.UserInviteToken alias Pleroma.UserInviteToken
describe "valid_invite?/1 one time invites" do describe "valid_invite?/1 one time invites" do
@ -64,7 +63,6 @@ defmodule Pleroma.UserInviteTokenTest do
test "expires yesterday returns false", %{invite: invite} do test "expires yesterday returns false", %{invite: invite} do
invite = %{invite | expires_at: Date.add(Date.utc_today(), -1)} invite = %{invite | expires_at: Date.add(Date.utc_today(), -1)}
invite = Repo.insert!(invite)
refute UserInviteToken.valid_invite?(invite) refute UserInviteToken.valid_invite?(invite)
end end
end end
@ -82,7 +80,6 @@ defmodule Pleroma.UserInviteTokenTest do
test "overdue date and less uses returns false", %{invite: invite} do test "overdue date and less uses returns false", %{invite: invite} do
invite = %{invite | expires_at: Date.add(Date.utc_today(), -1)} invite = %{invite | expires_at: Date.add(Date.utc_today(), -1)}
invite = Repo.insert!(invite)
refute UserInviteToken.valid_invite?(invite) refute UserInviteToken.valid_invite?(invite)
end end
@ -93,7 +90,6 @@ defmodule Pleroma.UserInviteTokenTest do
test "overdue date with more uses returns false", %{invite: invite} do test "overdue date with more uses returns false", %{invite: invite} do
invite = %{invite | expires_at: Date.add(Date.utc_today(), -1), uses: 5} invite = %{invite | expires_at: Date.add(Date.utc_today(), -1), uses: 5}
invite = Repo.insert!(invite)
refute UserInviteToken.valid_invite?(invite) refute UserInviteToken.valid_invite?(invite)
end end
end end

View file

@ -1,5 +1,5 @@
# Pleroma: A lightweight social networking server # Pleroma: A lightweight social networking server
# Copyright © 2019 Pleroma Authors <https://pleroma.social/> # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicyTest do defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicyTest do

View file

@ -1,5 +1,5 @@
# Pleroma: A lightweight social networking server # Pleroma: A lightweight social networking server
# Copyright © 2019 Pleroma Authors <https://pleroma.social/> # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicyTest do defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicyTest do
@ -110,6 +110,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicyTest do
end end
describe "with unknown actors" do describe "with unknown actors" do
setup do
Tesla.Mock.mock(fn
%{method: :get, url: "http://invalid.actor"} ->
%Tesla.Env{status: 500, body: ""}
end)
:ok
end
test "it rejects posts without links" do test "it rejects posts without links" do
message = message =
@linkless_message @linkless_message

View file

@ -1,5 +1,5 @@
# Pleroma: A lightweight social networking server # Pleroma: A lightweight social networking server
# Copyright © 2019 Pleroma Authors <https://pleroma.social/> # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrependedTest do defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrependedTest do

View file

@ -1,5 +1,5 @@
# Pleroma: A lightweight social networking server # Pleroma: A lightweight social networking server
# Copyright © 2019 Pleroma Authors <https://pleroma.social/> # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicyTest do defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicyTest do

View file

@ -1,5 +1,5 @@
# Pleroma: A lightweight social networking server # Pleroma: A lightweight social networking server
# Copyright © 2019 Pleroma Authors <https://pleroma.social/> # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkupTest do defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkupTest do

View file

@ -1,5 +1,5 @@
# Pleroma: A lightweight social networking server # Pleroma: A lightweight social networking server
# Copyright © 2019 Pleroma Authors <https://pleroma.social/> # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicyTest do defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicyTest do

View file

@ -1,5 +1,5 @@
# Pleroma: A lightweight social networking server # Pleroma: A lightweight social networking server
# Copyright © 2019 Pleroma Authors <https://pleroma.social/> # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublicTest do defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublicTest do

View file

@ -1,5 +1,5 @@
# Pleroma: A lightweight social networking server # Pleroma: A lightweight social networking server
# Copyright © 2019 Pleroma Authors <https://pleroma.social/> # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do

View file

@ -89,6 +89,11 @@ defmodule Pleroma.Web.ActivityPub.RelayTest do
} }
) )
Tesla.Mock.mock(fn
%{method: :get, url: "http://mastodon.example.org/eee/99541947525187367"} ->
%Tesla.Env{status: 500, body: ""}
end)
assert capture_log(fn -> assert capture_log(fn ->
assert Relay.publish(activity) == {:error, nil} assert Relay.publish(activity) == {:error, nil}
end) =~ "[error] error: nil" end) =~ "[error] error: nil"

View file

@ -2369,9 +2369,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
end end
test "common config example", %{conn: conn} do test "common config example", %{conn: conn} do
adapter = Application.get_env(:tesla, :adapter)
on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end)
conn = conn =
post(conn, "/api/pleroma/admin/config", %{ post(conn, "/api/pleroma/admin/config", %{
configs: [ configs: [
@ -2384,23 +2381,16 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
%{"tuple" => [":seconds_valid", 60]}, %{"tuple" => [":seconds_valid", 60]},
%{"tuple" => [":path", ""]}, %{"tuple" => [":path", ""]},
%{"tuple" => [":key1", nil]}, %{"tuple" => [":key1", nil]},
%{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]},
%{"tuple" => [":regex1", "~r/https:\/\/example.com/"]}, %{"tuple" => [":regex1", "~r/https:\/\/example.com/"]},
%{"tuple" => [":regex2", "~r/https:\/\/example.com/u"]}, %{"tuple" => [":regex2", "~r/https:\/\/example.com/u"]},
%{"tuple" => [":regex3", "~r/https:\/\/example.com/i"]}, %{"tuple" => [":regex3", "~r/https:\/\/example.com/i"]},
%{"tuple" => [":regex4", "~r/https:\/\/example.com/s"]}, %{"tuple" => [":regex4", "~r/https:\/\/example.com/s"]},
%{"tuple" => [":name", "Pleroma"]} %{"tuple" => [":name", "Pleroma"]}
] ]
},
%{
"group" => ":tesla",
"key" => ":adapter",
"value" => "Tesla.Adapter.Httpc"
} }
] ]
}) })
assert Application.get_env(:tesla, :adapter) == Tesla.Adapter.Httpc
assert Config.get([Pleroma.Captcha.NotReal, :name]) == "Pleroma" assert Config.get([Pleroma.Captcha.NotReal, :name]) == "Pleroma"
assert json_response(conn, 200) == %{ assert json_response(conn, 200) == %{
@ -2414,7 +2404,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
%{"tuple" => [":seconds_valid", 60]}, %{"tuple" => [":seconds_valid", 60]},
%{"tuple" => [":path", ""]}, %{"tuple" => [":path", ""]},
%{"tuple" => [":key1", nil]}, %{"tuple" => [":key1", nil]},
%{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]},
%{"tuple" => [":regex1", "~r/https:\\/\\/example.com/"]}, %{"tuple" => [":regex1", "~r/https:\\/\\/example.com/"]},
%{"tuple" => [":regex2", "~r/https:\\/\\/example.com/u"]}, %{"tuple" => [":regex2", "~r/https:\\/\\/example.com/u"]},
%{"tuple" => [":regex3", "~r/https:\\/\\/example.com/i"]}, %{"tuple" => [":regex3", "~r/https:\\/\\/example.com/i"]},
@ -2427,19 +2416,12 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
":seconds_valid", ":seconds_valid",
":path", ":path",
":key1", ":key1",
":partial_chain",
":regex1", ":regex1",
":regex2", ":regex2",
":regex3", ":regex3",
":regex4", ":regex4",
":name" ":name"
] ]
},
%{
"group" => ":tesla",
"key" => ":adapter",
"value" => "Tesla.Adapter.Httpc",
"db" => [":adapter"]
} }
] ]
} }

View file

@ -472,6 +472,13 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
activity = insert(:note_activity, user: user, note: object) activity = insert(:note_activity, user: user, note: object)
Pleroma.Repo.delete(object) Pleroma.Repo.delete(object)
obj_url = activity.data["object"]
Tesla.Mock.mock(fn
%{method: :get, url: ^obj_url} ->
%Tesla.Env{status: 404, body: ""}
end)
assert Utils.maybe_notify_mentioned_recipients(["test-test"], activity) == [ assert Utils.maybe_notify_mentioned_recipients(["test-test"], activity) == [
"test-test" "test-test"
] ]

View file

@ -472,11 +472,24 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do
assert length(json_response(conn, 200)) == 1 assert length(json_response(conn, 200)) == 1
end end
@tag capture_log: true
test "see move notifications" do test "see move notifications" do
old_user = insert(:user) old_user = insert(:user)
new_user = insert(:user, also_known_as: [old_user.ap_id]) new_user = insert(:user, also_known_as: [old_user.ap_id])
%{user: follower, conn: conn} = oauth_access(["read:notifications"]) %{user: follower, conn: conn} = oauth_access(["read:notifications"])
old_user_url = old_user.ap_id
body =
File.read!("test/fixtures/users_mock/localhost.json")
|> String.replace("{{nickname}}", old_user.nickname)
|> Jason.encode!()
Tesla.Mock.mock(fn
%{method: :get, url: ^old_user_url} ->
%Tesla.Env{status: 200, body: body}
end)
User.follow(follower, old_user) User.follow(follower, old_user)
Pleroma.Web.ActivityPub.ActivityPub.move(old_user, new_user) Pleroma.Web.ActivityPub.ActivityPub.move(old_user, new_user)
Pleroma.Tests.ObanHelpers.perform_all() Pleroma.Tests.ObanHelpers.perform_all()

View file

@ -112,11 +112,24 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do
test_notifications_rendering([notification], followed, []) test_notifications_rendering([notification], followed, [])
end end
@tag capture_log: true
test "Move notification" do test "Move notification" do
old_user = insert(:user) old_user = insert(:user)
new_user = insert(:user, also_known_as: [old_user.ap_id]) new_user = insert(:user, also_known_as: [old_user.ap_id])
follower = insert(:user) follower = insert(:user)
old_user_url = old_user.ap_id
body =
File.read!("test/fixtures/users_mock/localhost.json")
|> String.replace("{{nickname}}", old_user.nickname)
|> Jason.encode!()
Tesla.Mock.mock(fn
%{method: :get, url: ^old_user_url} ->
%Tesla.Env{status: 200, body: body}
end)
User.follow(follower, old_user) User.follow(follower, old_user)
Pleroma.Web.ActivityPub.ActivityPub.move(old_user, new_user) Pleroma.Web.ActivityPub.ActivityPub.move(old_user, new_user)
Pleroma.Tests.ObanHelpers.perform_all() Pleroma.Tests.ObanHelpers.perform_all()

View file

@ -94,6 +94,23 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
Repo.delete(user) Repo.delete(user)
Cachex.clear(:user_cache) Cachex.clear(:user_cache)
finger_url =
"https://localhost/.well-known/webfinger?resource=acct:#{user.nickname}@localhost"
Tesla.Mock.mock_global(fn
%{method: :get, url: "http://localhost/.well-known/host-meta"} ->
%Tesla.Env{status: 404, body: ""}
%{method: :get, url: "https://localhost/.well-known/host-meta"} ->
%Tesla.Env{status: 404, body: ""}
%{
method: :get,
url: ^finger_url
} ->
%Tesla.Env{status: 404, body: ""}
end)
%{account: ms_user} = StatusView.render("show.json", activity: activity) %{account: ms_user} = StatusView.render("show.json", activity: activity)
assert ms_user.acct == "erroruser@example.com" assert ms_user.acct == "erroruser@example.com"

View file

@ -134,7 +134,7 @@ defmodule Pleroma.Web.Push.ImplTest do
user = insert(:user, nickname: "Bob") user = insert(:user, nickname: "Bob")
other_user = insert(:user) other_user = insert(:user)
{:ok, _, _, activity} = CommonAPI.follow(user, other_user) {:ok, _, _, activity} = CommonAPI.follow(user, other_user)
object = Object.normalize(activity) object = Object.normalize(activity, false)
assert Impl.format_body(%{activity: activity}, user, object) == "@Bob has followed you" assert Impl.format_body(%{activity: activity}, user, object) == "@Bob has followed you"

View file

@ -121,6 +121,18 @@ defmodule Pleroma.Web.StreamerTest do
test "it sends follow activities to the 'user:notification' stream", %{ test "it sends follow activities to the 'user:notification' stream", %{
user: user user: user
} do } do
user_url = user.ap_id
body =
File.read!("test/fixtures/users_mock/localhost.json")
|> String.replace("{{nickname}}", user.nickname)
|> Jason.encode!()
Tesla.Mock.mock_global(fn
%{method: :get, url: ^user_url} ->
%Tesla.Env{status: 200, body: body}
end)
user2 = insert(:user) user2 = insert(:user)
task = Task.async(fn -> assert_receive {:text, _}, @streamer_timeout end) task = Task.async(fn -> assert_receive {:text, _}, @streamer_timeout end)