From 400337b0a78150704de650d62a499464e895816c Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 1 Jan 2019 14:46:55 +0100 Subject: [PATCH 1/2] Make Federator options configurable. --- config/config.exs | 8 ++++++++ lib/pleroma/web/federator/federator.ex | 3 +-- lib/pleroma/web/federator/retry_queue.ex | 19 +++++++------------ 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/config/config.exs b/config/config.exs index 90e3a4aec..d30b33197 100644 --- a/config/config.exs +++ b/config/config.exs @@ -252,6 +252,14 @@ config :pleroma, Pleroma.User, "internal" ] +config :pleroma, Pleroma.Web.Federator, max_jobs: 50 + +config :pleroma, Pleroma.Web.Federator.RetryQueue, + enabled: false, + max_jobs: 20, + initial_timeout: 30, + max_retries: 5 + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index d25bfc0c1..f3a0e18b8 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -17,7 +17,6 @@ defmodule Pleroma.Web.Federator do @websub Application.get_env(:pleroma, :websub) @ostatus Application.get_env(:pleroma, :ostatus) - @max_jobs 20 def init(args) do {:ok, args} @@ -168,7 +167,7 @@ defmodule Pleroma.Web.Federator do end def maybe_start_job(running_jobs, queue) do - if :sets.size(running_jobs) < @max_jobs && queue != [] do + if :sets.size(running_jobs) < Pleroma.Config.get([__MODULE__, :max_jobs]) && queue != [] do {{type, payload}, queue} = queue_pop(queue) {:ok, pid} = Task.start(fn -> handle(type, payload) end) mref = Process.monitor(pid) diff --git a/lib/pleroma/web/federator/retry_queue.ex b/lib/pleroma/web/federator/retry_queue.ex index c57bbb77c..230a2c939 100644 --- a/lib/pleroma/web/federator/retry_queue.ex +++ b/lib/pleroma/web/federator/retry_queue.ex @@ -7,12 +7,6 @@ defmodule Pleroma.Web.Federator.RetryQueue do require Logger - # seconds - @initial_timeout 30 - @max_retries 5 - - @max_jobs 20 - def init(args) do queue_table = :ets.new(:pleroma_retry_queue, [:bag, :protected]) @@ -21,7 +15,7 @@ defmodule Pleroma.Web.Federator.RetryQueue do def start_link() do enabled = - if Mix.env() == :test, do: true, else: Pleroma.Config.get([:retry_queue, :enabled], false) + if Mix.env() == :test, do: true, else: Pleroma.Config.get([__MODULE__, :enabled], false) if enabled do Logger.info("Starting retry queue") @@ -54,7 +48,7 @@ defmodule Pleroma.Web.Federator.RetryQueue do end def get_retry_params(retries) do - if retries > @max_retries do + if retries > Pleroma.Config.get([__MODULE__, :max_retries]) do {:drop, "Max retries reached"} else {:retry, growth_function(retries)} @@ -108,12 +102,12 @@ defmodule Pleroma.Web.Federator.RetryQueue do current_time = DateTime.to_unix(DateTime.utc_now()) n_running_jobs = :sets.size(running_jobs) - if n_running_jobs < @max_jobs do + if n_running_jobs < Pleroma.Config.get([__MODULE__, :max_jobs]) do n_ready_jobs = ets_count_expires(queue_table, current_time) if n_ready_jobs > 0 do # figure out how many we could start - available_job_slots = @max_jobs - n_running_jobs + available_job_slots = Pleroma.Config.get([__MODULE__, :max_jobs]) - n_running_jobs start_n_jobs(running_jobs, queue_table, current_time, available_job_slots) else running_jobs @@ -228,12 +222,13 @@ defmodule Pleroma.Web.Federator.RetryQueue do if Mix.env() == :test do defp growth_function(_retries) do - _shutit = @initial_timeout + _shutit = Pleroma.Config.get([__MODULE__, :initial_timeout]) DateTime.to_unix(DateTime.utc_now()) - 1 end else defp growth_function(retries) do - round(@initial_timeout * :math.pow(retries, 3)) + DateTime.to_unix(DateTime.utc_now()) + round(Pleroma.Config.get([__MODULE__, :initial_timeout]) * :math.pow(retries, 3)) + + DateTime.to_unix(DateTime.utc_now()) end end From 1a0391c8a474c282823d3b2ab7a0cae62328c5b3 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 1 Jan 2019 17:40:42 +0100 Subject: [PATCH 2/2] Add documentation. --- docs/config.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/config.md b/docs/config.md index 0aeaf934e..f4bcae3fd 100644 --- a/docs/config.md +++ b/docs/config.md @@ -193,3 +193,14 @@ You can then do ``` curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerandomtoken" ``` + +## Pleroma.Web.Federator + +* `max_jobs`: The maximum amount of parallel federation jobs running at the same time. + +## Pleroma.Web.Federator.RetryQueue + +* `enabled`: If set to `true`, failed federation jobs will be retried +* `max_jobs`: The maximum amount of parallel federation jbos running at the same time. +* `initial_timeout`: The initial timeout in seconds +* `max_retries`: The maximum number of times a federation job is retried