From b4ff63d020293bd633bc9c01af1078cacf7f90ed Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Sat, 9 Jan 2021 18:52:40 +0300 Subject: [PATCH 1/5] configurable limits for ConcurrentLimiter Pleroma.Web.RichMedia.Helpers & Pleroma.Web.MediaProxy --- config/config.exs | 5 ++++ config/description.exs | 48 ++++++++++++++++++++++++++++++++ docs/configuration/cheatsheet.md | 12 ++++++++ lib/pleroma/application.ex | 11 +++++++- 4 files changed, 75 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 70d0c2c2b..e07e67de9 100644 --- a/config/config.exs +++ b/config/config.exs @@ -832,6 +832,11 @@ limit_days: 7, dir: nil +config :pleroma, ConcurrentLimiter, [ + {Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]}, + {Pleroma.Web.MediaProxy, [max_running: 5, max_waiting: 5]} +] + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/config/description.exs b/config/description.exs index 493d362d3..49fea4234 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3330,5 +3330,53 @@ suggestions: [:text, :protobuf] } ] + }, + %{ + group: :pleroma, + key: ConcurrentLimiter, + type: :group, + description: "Limits configuration for background tasks.", + children: [ + %{ + key: Pleroma.Web.RichMedia.Helpers, + type: :keyword, + description: "Concurrent limits configuration for getting RichMedia for activities.", + suggestions: [max_running: 5, max_waiting: 5], + children: [ + %{ + key: :max_running, + type: :integer, + description: "Max running concurrently jobs.", + suggestion: [5] + }, + %{ + key: :max_waiting, + type: :integer, + description: "Max waiting jobs.", + suggestion: [5] + } + ] + }, + %{ + key: Pleroma.Web.MediaProxy, + type: :keyword, + description: "Concurrent limits configuration for MediaProxyWarmingPolicy.", + suggestions: [max_running: 5, max_waiting: 5], + children: [ + %{ + key: :max_running, + type: :integer, + description: "Max running concurrently jobs.", + suggestion: [5] + }, + %{ + key: :max_waiting, + type: :integer, + description: "Max waiting jobs.", + suggestion: [5] + } + ] + } + ] } ] diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index c7d8a2dae..c7ff8687e 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -1110,3 +1110,15 @@ Settings to enable and configure expiration for ephemeral activities * `:enabled` - enables ephemeral activities creation * `:min_lifetime` - minimum lifetime for ephemeral activities (in seconds). Default: 10 minutes. + +## ConcurrentLimiter + +Settings allow configuring restrictions for concurrently running jobs. Jobs, which can be configured: + +* `Pleroma.Web.RichMedia.Helpers` - configuration for getting RichMedia for activities. +* `Pleroma.Web.MediaProxy` - configuration for MediaProxyWarmingPolicy. + +Each job has these settings: + +* `:max_running` - max concurrently runnings jobs +* `:max_waiting` - max waiting jobs diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 203a95004..4742a3ecb 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -297,7 +297,16 @@ defp http_children(_, _), do: [] @spec limiters_setup() :: :ok def limiters_setup do + config = Config.get(ConcurrentLimiter, []) + [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.MediaProxy] - |> Enum.each(&ConcurrentLimiter.new(&1, 1, 0)) + |> Enum.each(fn module -> + mod_config = Keyword.get(config, module, []) + + max_running = Keyword.get(mod_config, :max_running, 5) + max_waiting = Keyword.get(mod_config, :max_waiting, 5) + + ConcurrentLimiter.new(module, max_running, max_waiting) + end) end end From 1537a4f0adfdc079d7d77dbe249c83df5c3b2eef Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Wed, 20 Jan 2021 17:01:26 -0600 Subject: [PATCH 2/5] Document ConcurrentLimiter for RichMedia and MediaProxy --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76eab51d4..2ab432d3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - OAuth form improvements: users are remembered by their cookie, the CSS is overridable by the admin, and the style has been improved. - OAuth improvements and fixes: more secure session-based authentication (by token that could be revoked anytime), ability to revoke belonging OAuth token from any client etc. - Ability to set ActivityPub aliases for follower migration. +- Configurable background job limits for RichMedia (link previews) and MediaProxy + <details> <summary>API Changes</summary> From dece31a031b8fce5b47c61ad014aa38ae72ee685 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@feld.me> Date: Wed, 20 Jan 2021 17:07:00 -0600 Subject: [PATCH 3/5] Update docs --- docs/configuration/cheatsheet.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index c7ff8687e..e7a1b40b1 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -1113,10 +1113,10 @@ Settings to enable and configure expiration for ephemeral activities ## ConcurrentLimiter -Settings allow configuring restrictions for concurrently running jobs. Jobs, which can be configured: +Settings to restrict concurrently running jobs. Jobs which can be configured: -* `Pleroma.Web.RichMedia.Helpers` - configuration for getting RichMedia for activities. -* `Pleroma.Web.MediaProxy` - configuration for MediaProxyWarmingPolicy. +* `Pleroma.Web.RichMedia.Helpers` - generating link previews of URLs in activities +* `Pleroma.Web.MediaProxy` - fetching remote media via MediaProxy Each job has these settings: From 6d48144a9d7273e1b6c253164af5550580a6ea9f Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Thu, 21 Jan 2021 09:50:18 +0300 Subject: [PATCH 4/5] use proper naming for MediaProxyWarmingPolicy in ConcurrentLimiter --- config/description.exs | 2 +- docs/configuration/cheatsheet.md | 2 +- lib/pleroma/application.ex | 2 +- lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/description.exs b/config/description.exs index 49fea4234..715a0d0c3 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3358,7 +3358,7 @@ ] }, %{ - key: Pleroma.Web.MediaProxy, + key: Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, type: :keyword, description: "Concurrent limits configuration for MediaProxyWarmingPolicy.", suggestions: [max_running: 5, max_waiting: 5], diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index e7a1b40b1..5c0fd6487 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -1116,7 +1116,7 @@ Settings to enable and configure expiration for ephemeral activities Settings to restrict concurrently running jobs. Jobs which can be configured: * `Pleroma.Web.RichMedia.Helpers` - generating link previews of URLs in activities -* `Pleroma.Web.MediaProxy` - fetching remote media via MediaProxy +* `Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy` - warming remote media cache via MediaProxyWarmingPolicy Each job has these settings: diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 4742a3ecb..9e262235e 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -299,7 +299,7 @@ defp http_children(_, _), do: [] def limiters_setup do config = Config.get(ConcurrentLimiter, []) - [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.MediaProxy] + [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy] |> Enum.each(fn module -> mod_config = Keyword.get(config, module, []) diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index 50d48edc8..8dbf44071 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -27,7 +27,7 @@ defp prefetch(url) do if Pleroma.Config.get(:env) == :test do fetch(prefetch_url) else - ConcurrentLimiter.limit(MediaProxy, fn -> + ConcurrentLimiter.limit(__MODULE__, fn -> Task.start(fn -> fetch(prefetch_url) end) end) end From 5ade430e46e76543b317dc07fdbc0a3fe7367621 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Thu, 21 Jan 2021 10:12:01 +0300 Subject: [PATCH 5/5] changed naming in changelog --- CHANGELOG.md | 2 +- config/config.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ab432d3f..e1dfeae01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - OAuth form improvements: users are remembered by their cookie, the CSS is overridable by the admin, and the style has been improved. - OAuth improvements and fixes: more secure session-based authentication (by token that could be revoked anytime), ability to revoke belonging OAuth token from any client etc. - Ability to set ActivityPub aliases for follower migration. -- Configurable background job limits for RichMedia (link previews) and MediaProxy +- Configurable background job limits for RichMedia (link previews) and MediaProxyWarmingPolicy <details> diff --git a/config/config.exs b/config/config.exs index e07e67de9..c4a690799 100644 --- a/config/config.exs +++ b/config/config.exs @@ -834,7 +834,7 @@ config :pleroma, ConcurrentLimiter, [ {Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]}, - {Pleroma.Web.MediaProxy, [max_running: 5, max_waiting: 5]} + {Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]} ] # Import environment specific config. This must remain at the bottom