Merge branch 'limiter-setup-fix' into 'develop'

Configurable limits for ConcurrentLimiter for Pleroma.Web.RichMedia.Helpers & Pleroma.Web.MediaProxyWarmingPolicy

See merge request pleroma/pleroma!3248

CHANGELOG.md

@@ -34,6 +34,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - OAuth form improvements: users are remembered by their cookie, the CSS is overridable by the admin, and the style has been improved.
 - OAuth improvements and fixes: more secure session-based authentication (by token that could be revoked anytime), ability to revoke belonging OAuth token from any client etc.
 - Ability to set ActivityPub aliases for follower migration.
+- Configurable background job limits for RichMedia (link previews) and MediaProxyWarmingPolicy
+
 
 <details>
   <summary>API Changes</summary>

config/config.exs

@@ -832,6 +832,11 @@
   limit_days: 7,
   dir: nil
 
+config :pleroma, ConcurrentLimiter, [
+  {Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]},
+  {Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]}
+]
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

config/description.exs

@@ -3330,5 +3330,53 @@
         suggestions: [:text, :protobuf]
       }
     ]
+  },
+  %{
+    group: :pleroma,
+    key: ConcurrentLimiter,
+    type: :group,
+    description: "Limits configuration for background tasks.",
+    children: [
+      %{
+        key: Pleroma.Web.RichMedia.Helpers,
+        type: :keyword,
+        description: "Concurrent limits configuration for getting RichMedia for activities.",
+        suggestions: [max_running: 5, max_waiting: 5],
+        children: [
+          %{
+            key: :max_running,
+            type: :integer,
+            description: "Max running concurrently jobs.",
+            suggestion: [5]
+          },
+          %{
+            key: :max_waiting,
+            type: :integer,
+            description: "Max waiting jobs.",
+            suggestion: [5]
+          }
+        ]
+      },
+      %{
+        key: Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy,
+        type: :keyword,
+        description: "Concurrent limits configuration for MediaProxyWarmingPolicy.",
+        suggestions: [max_running: 5, max_waiting: 5],
+        children: [
+          %{
+            key: :max_running,
+            type: :integer,
+            description: "Max running concurrently jobs.",
+            suggestion: [5]
+          },
+          %{
+            key: :max_waiting,
+            type: :integer,
+            description: "Max waiting jobs.",
+            suggestion: [5]
+          }
+        ]
+      }
+    ]
   }
 ]

docs/configuration/cheatsheet.md

@@ -1110,3 +1110,15 @@ Settings to enable and configure expiration for ephemeral activities
 
 * `:enabled` - enables ephemeral activities creation
 * `:min_lifetime` - minimum lifetime for ephemeral activities (in seconds). Default: 10 minutes.
+
+## ConcurrentLimiter
+
+Settings to restrict concurrently running jobs. Jobs which can be configured:
+
+* `Pleroma.Web.RichMedia.Helpers` - generating link previews of URLs in activities
+* `Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy` - warming remote media cache via MediaProxyWarmingPolicy
+
+Each job has these settings:
+
+* `:max_running` - max concurrently runnings jobs
+* `:max_waiting` - max waiting jobs

lib/pleroma/application.ex

@@ -297,7 +297,16 @@ defp http_children(_, _), do: []
 
   @spec limiters_setup() :: :ok
   def limiters_setup do
-    [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.MediaProxy]
+    config = Config.get(ConcurrentLimiter, [])
-    |> Enum.each(&ConcurrentLimiter.new(&1, 1, 0))
+
+    [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy]
+    |> Enum.each(fn module ->
+      mod_config = Keyword.get(config, module, [])
+
+      max_running = Keyword.get(mod_config, :max_running, 5)
+      max_waiting = Keyword.get(mod_config, :max_waiting, 5)
+
+      ConcurrentLimiter.new(module, max_running, max_waiting)
+    end)
   end
 end

lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex

@@ -27,7 +27,7 @@ defp prefetch(url) do
       if Pleroma.Config.get(:env) == :test do
         fetch(prefetch_url)
       else
-        ConcurrentLimiter.limit(MediaProxy, fn ->
+        ConcurrentLimiter.limit(__MODULE__, fn ->
           Task.start(fn -> fetch(prefetch_url) end)
         end)
       end