Merge branch 'limiter-setup-fix' into 'develop'

Configurable limits for ConcurrentLimiter for Pleroma.Web.RichMedia.Helpers & Pleroma.Web.MediaProxyWarmingPolicy

See merge request pleroma/pleroma!3248
This commit is contained in:
feld 2021-01-21 14:59:51 +00:00
commit d8860eaee4
6 changed files with 79 additions and 3 deletions

View file

@ -34,6 +34,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- OAuth form improvements: users are remembered by their cookie, the CSS is overridable by the admin, and the style has been improved. - OAuth form improvements: users are remembered by their cookie, the CSS is overridable by the admin, and the style has been improved.
- OAuth improvements and fixes: more secure session-based authentication (by token that could be revoked anytime), ability to revoke belonging OAuth token from any client etc. - OAuth improvements and fixes: more secure session-based authentication (by token that could be revoked anytime), ability to revoke belonging OAuth token from any client etc.
- Ability to set ActivityPub aliases for follower migration. - Ability to set ActivityPub aliases for follower migration.
- Configurable background job limits for RichMedia (link previews) and MediaProxyWarmingPolicy
<details> <details>
<summary>API Changes</summary> <summary>API Changes</summary>

View file

@ -832,6 +832,11 @@
limit_days: 7, limit_days: 7,
dir: nil dir: nil
config :pleroma, ConcurrentLimiter, [
{Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]},
{Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]}
]
# Import environment specific config. This must remain at the bottom # Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above. # of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs" import_config "#{Mix.env()}.exs"

View file

@ -3330,5 +3330,53 @@
suggestions: [:text, :protobuf] suggestions: [:text, :protobuf]
} }
] ]
},
%{
group: :pleroma,
key: ConcurrentLimiter,
type: :group,
description: "Limits configuration for background tasks.",
children: [
%{
key: Pleroma.Web.RichMedia.Helpers,
type: :keyword,
description: "Concurrent limits configuration for getting RichMedia for activities.",
suggestions: [max_running: 5, max_waiting: 5],
children: [
%{
key: :max_running,
type: :integer,
description: "Max running concurrently jobs.",
suggestion: [5]
},
%{
key: :max_waiting,
type: :integer,
description: "Max waiting jobs.",
suggestion: [5]
}
]
},
%{
key: Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy,
type: :keyword,
description: "Concurrent limits configuration for MediaProxyWarmingPolicy.",
suggestions: [max_running: 5, max_waiting: 5],
children: [
%{
key: :max_running,
type: :integer,
description: "Max running concurrently jobs.",
suggestion: [5]
},
%{
key: :max_waiting,
type: :integer,
description: "Max waiting jobs.",
suggestion: [5]
}
]
}
]
} }
] ]

View file

@ -1110,3 +1110,15 @@ Settings to enable and configure expiration for ephemeral activities
* `:enabled` - enables ephemeral activities creation * `:enabled` - enables ephemeral activities creation
* `:min_lifetime` - minimum lifetime for ephemeral activities (in seconds). Default: 10 minutes. * `:min_lifetime` - minimum lifetime for ephemeral activities (in seconds). Default: 10 minutes.
## ConcurrentLimiter
Settings to restrict concurrently running jobs. Jobs which can be configured:
* `Pleroma.Web.RichMedia.Helpers` - generating link previews of URLs in activities
* `Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy` - warming remote media cache via MediaProxyWarmingPolicy
Each job has these settings:
* `:max_running` - max concurrently runnings jobs
* `:max_waiting` - max waiting jobs

View file

@ -297,7 +297,16 @@ defp http_children(_, _), do: []
@spec limiters_setup() :: :ok @spec limiters_setup() :: :ok
def limiters_setup do def limiters_setup do
[Pleroma.Web.RichMedia.Helpers, Pleroma.Web.MediaProxy] config = Config.get(ConcurrentLimiter, [])
|> Enum.each(&ConcurrentLimiter.new(&1, 1, 0))
[Pleroma.Web.RichMedia.Helpers, Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy]
|> Enum.each(fn module ->
mod_config = Keyword.get(config, module, [])
max_running = Keyword.get(mod_config, :max_running, 5)
max_waiting = Keyword.get(mod_config, :max_waiting, 5)
ConcurrentLimiter.new(module, max_running, max_waiting)
end)
end end
end end

View file

@ -27,7 +27,7 @@ defp prefetch(url) do
if Pleroma.Config.get(:env) == :test do if Pleroma.Config.get(:env) == :test do
fetch(prefetch_url) fetch(prefetch_url)
else else
ConcurrentLimiter.limit(MediaProxy, fn -> ConcurrentLimiter.limit(__MODULE__, fn ->
Task.start(fn -> fetch(prefetch_url) end) Task.start(fn -> fetch(prefetch_url) end)
end) end)
end end