From b91e671c0d7cdca6b1ddf753f2017b15f366695f Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 16 Dec 2022 17:22:26 +0000 Subject: [PATCH 1/4] add remote user count for the heck of it --- lib/pleroma/stats.ex | 12 +++++++++++- lib/pleroma/web/telemetry.ex | 4 +++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 3e3f24c2c..cda1fb9a6 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -85,14 +85,24 @@ def calculate_stat_data do where: not u.invisible ) + remote_users_query = + from(u in User, + where: u.is_active == true, + where: u.local == false, + where: not is_nil(u.nickname), + where: not u.invisible + ) + user_count = Repo.aggregate(users_query, :count, :id) + remote_user_count = Repo.aggregate(remote_users_query, :count, :id) %{ peers: peers, stats: %{ domain_count: domain_count, status_count: status_count || 0, - user_count: user_count + user_count: user_count, + remote_user_count: remote_user_count } } end diff --git a/lib/pleroma/web/telemetry.ex b/lib/pleroma/web/telemetry.ex index 5b01ee14d..acb649421 100644 --- a/lib/pleroma/web/telemetry.ex +++ b/lib/pleroma/web/telemetry.ex @@ -109,7 +109,8 @@ defp summary_metrics do summary("vm.total_run_queue_lengths.io"), last_value("pleroma.local_users.total"), last_value("pleroma.domains.total"), - last_value("pleroma.local_statuses.total") + last_value("pleroma.local_statuses.total"), + last_value("pleroma.remote_users.total") ] end @@ -127,5 +128,6 @@ def instance_stats do :telemetry.execute([:pleroma, :local_users], %{total: stats.user_count}, %{}) :telemetry.execute([:pleroma, :domains], %{total: stats.domain_count}, %{}) :telemetry.execute([:pleroma, :local_statuses], %{total: stats.status_count}, %{}) + :telemetry.execute([:pleroma, :remote_users], %{total: stats.remote_user_count}, %{}) end end From 7b76fdeed3edce937514f25b818a2d4ed46f7892 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 16 Dec 2022 17:22:56 +0000 Subject: [PATCH 2/4] update stats every 5 minutes --- lib/pleroma/stats.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index cda1fb9a6..c47a0f9de 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Stats do alias Pleroma.Repo alias Pleroma.User - @interval :timer.seconds(60) + @interval :timer.seconds(300) def start_link(_) do GenServer.start_link( From 126f1ca69c7b54dd8fdecafa291df21153ad80f2 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 16 Dec 2022 17:31:04 +0000 Subject: [PATCH 3/4] increase rich media backoff time --- CHANGELOG.md | 1 + config/config.exs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4235f6dab..c69eb4c82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - Return HTTP error 413 when uploading an avatar or banner that's above the configured upload limit instead of a 500. - Non-admin users now cannot register `admin` scope tokens (not security-critical, they didn't work before, but you _could_ create them) +- Rich media will now backoff for 20 minutes after a failure ### Upgrade notes - Ensure `config :tesla, :adapter` is either unset, or set to `{Tesla.Adapter.Finch, name: MyFinch}` in your .exs config diff --git a/config/config.exs b/config/config.exs index 2d13fa500..4c39d52b3 100644 --- a/config/config.exs +++ b/config/config.exs @@ -425,7 +425,7 @@ Pleroma.Web.RichMedia.Parsers.TwitterCard, Pleroma.Web.RichMedia.Parsers.OEmbed ], - failure_backoff: 60_000, + failure_backoff: :timer.minutes(20), ttl_setters: [Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl] config :pleroma, :media_proxy, From dcac8adb3de7fb5545f6c6f18c9e1d5d82606594 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 16 Dec 2022 18:33:00 +0000 Subject: [PATCH 4/4] Add option to modify HTTP pool size --- CHANGELOG.md | 1 + config/config.exs | 1 + config/description.exs | 7 +++++++ lib/pleroma/application.ex | 2 ++ lib/pleroma/config/transfer_task.ex | 4 +++- lib/pleroma/http/adapter_helper.ex | 7 +++++++ test/instance_static/emoji/dump_pack/pack.json | 6 ------ test/pleroma/http/adapter_helper_test.exs | 7 +++++++ 8 files changed, 28 insertions(+), 7 deletions(-) delete mode 100644 test/instance_static/emoji/dump_pack/pack.json diff --git a/CHANGELOG.md b/CHANGELOG.md index c69eb4c82..6dcbc4b14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - Prometheus metrics exporting from `/api/v1/akkoma/metrics` +- Ability to alter http pool size ### Removed - Non-finch HTTP adapters diff --git a/config/config.exs b/config/config.exs index 4c39d52b3..48290fb05 100644 --- a/config/config.exs +++ b/config/config.exs @@ -179,6 +179,7 @@ receive_timeout: :timer.seconds(15), proxy_url: nil, user_agent: :default, + pool_size: 50, adapter: [] config :pleroma, :instance, diff --git a/config/description.exs b/config/description.exs index 1059039e7..eb61c7218 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2661,6 +2661,13 @@ "What user agent to use. Must be a string or an atom `:default`. Default value is `:default`.", suggestions: ["Pleroma", :default] }, + %{ + key: :pool_size, + type: :integer, + description: + "Number of concurrent outbound HTTP requests to allow. Default 50.", + suggestions: [50] + }, %{ key: :adapter, type: :keyword, diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 02336d6d1..26b500dc8 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -259,10 +259,12 @@ def limiters_setup do defp http_children do proxy_url = Config.get([:http, :proxy_url]) proxy = Pleroma.HTTP.AdapterHelper.format_proxy(proxy_url) + pool_size = Config.get([:http, :pool_size]) config = [:http, :adapter] |> Config.get([]) + |> Pleroma.HTTP.AdapterHelper.add_pool_size(pool_size) |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy_pool(proxy) |> Keyword.put(:name, MyFinch) diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 2e4bcb0ee..4fcaab4a5 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -25,7 +25,9 @@ defp reboot_time_subkeys, do: [ {:pleroma, Pleroma.Captcha, [:seconds_valid]}, {:pleroma, Pleroma.Upload, [:proxy_remote]}, - {:pleroma, :instance, [:upload_limit]} + {:pleroma, :instance, [:upload_limit]}, + {:pleroma, :http, [:pool_size]}, + {:pleroma, :http, [:proxy_url]} ] def start_link(restart_pleroma? \\ true) do diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index 77b854b5d..e5da3ffa8 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -47,6 +47,13 @@ def maybe_add_proxy_pool(opts, proxy) do |> put_in([:pools, :default, :conn_opts, :proxy], proxy) end + def add_pool_size(opts, pool_size) do + opts + |> maybe_add_pools() + |> maybe_add_default_pool() + |> put_in([:pools, :default, :size], pool_size) + end + defp maybe_add_pools(opts) do if Keyword.has_key?(opts, :pools) do opts diff --git a/test/instance_static/emoji/dump_pack/pack.json b/test/instance_static/emoji/dump_pack/pack.json deleted file mode 100644 index 3ca753be0..000000000 --- a/test/instance_static/emoji/dump_pack/pack.json +++ /dev/null @@ -1,6 +0,0 @@ -{ -"files": { }, -"pack": { -"description": "Dump pack", "homepage": "https://pleroma.social", -"license": "Test license", "share-files": true -}} diff --git a/test/pleroma/http/adapter_helper_test.exs b/test/pleroma/http/adapter_helper_test.exs index ba09f3422..cd7491e9a 100644 --- a/test/pleroma/http/adapter_helper_test.exs +++ b/test/pleroma/http/adapter_helper_test.exs @@ -66,4 +66,11 @@ test "receive_timeout should be overridden by :http, :receive_timeout" do assert options[:receive_timeout] == 20_000 end end + + describe "pool size settings" do + test "should get set" do + options = AdapterHelper.add_pool_size([], 50) + assert options[:pools][:default][:size] == 50 + end + end end