From 51bdf0cab6dc96bfd48a6d98d9f21584b42c0e44 Mon Sep 17 00:00:00 2001 From: stwf Date: Wed, 14 Aug 2019 11:55:17 -0400 Subject: [PATCH 1/5] use default child_specs --- lib/pleroma/application.ex | 185 ++++++-------------- lib/pleroma/captcha/captcha.ex | 2 +- lib/pleroma/config/transfer_task.ex | 2 +- lib/pleroma/emoji.ex | 2 +- lib/pleroma/flake_id.ex | 2 +- lib/pleroma/gopher/server.ex | 2 +- lib/pleroma/scheduled_activity_worker.ex | 2 +- lib/pleroma/stats.ex | 4 +- lib/pleroma/web/chat_channel.ex | 4 +- lib/pleroma/web/federator/retry_queue.ex | 2 +- lib/pleroma/web/oauth/token/clean_worker.ex | 3 +- lib/pleroma/web/streamer.ex | 2 +- test/config/transfer_task_test.exs | 4 +- 13 files changed, 73 insertions(+), 143 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 00b06f723..3bb0718e4 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -3,11 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Application do + import Cachex.Spec use Application @name Mix.Project.config()[:name] @version Mix.Project.config()[:version] @repository Mix.Project.config()[:source_url] + @env Mix.env() + def name, do: @name def version, do: @version def named_version, do: @name <> " " <> @version @@ -21,116 +24,25 @@ def user_agent do # See http://elixir-lang.org/docs/stable/elixir/Application.html # for more information on OTP Applications def start(_type, _args) do - import Cachex.Spec - Pleroma.Config.DeprecationWarnings.warn() setup_instrumenters() # Define workers and child supervisors to be supervised children = [ - # Start the Ecto repository - %{id: Pleroma.Repo, start: {Pleroma.Repo, :start_link, []}, type: :supervisor}, - %{id: Pleroma.Config.TransferTask, start: {Pleroma.Config.TransferTask, :start_link, []}}, - %{id: Pleroma.Emoji, start: {Pleroma.Emoji, :start_link, []}}, - %{id: Pleroma.Captcha, start: {Pleroma.Captcha, :start_link, []}}, - %{ - id: :cachex_used_captcha_cache, - start: - {Cachex, :start_link, - [ - :used_captcha_cache, - [ - ttl_interval: - :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid])) - ] - ]} - }, - %{ - id: :cachex_user, - start: - {Cachex, :start_link, - [ - :user_cache, - [ - default_ttl: 25_000, - ttl_interval: 1000, - limit: 2500 - ] - ]} - }, - %{ - id: :cachex_object, - start: - {Cachex, :start_link, - [ - :object_cache, - [ - default_ttl: 25_000, - ttl_interval: 1000, - limit: 2500 - ] - ]} - }, - %{ - id: :cachex_rich_media, - start: - {Cachex, :start_link, - [ - :rich_media_cache, - [ - default_ttl: :timer.minutes(120), - limit: 5000 - ] - ]} - }, - %{ - id: :cachex_scrubber, - start: - {Cachex, :start_link, - [ - :scrubber_cache, - [ - limit: 2500 - ] - ]} - }, - %{ - id: :cachex_idem, - start: - {Cachex, :start_link, - [ - :idempotency_cache, - [ - expiration: - expiration( - default: :timer.seconds(6 * 60 * 60), - interval: :timer.seconds(60) - ), - limit: 2500 - ] - ]} - }, - %{id: Pleroma.FlakeId, start: {Pleroma.FlakeId, :start_link, []}}, - %{ - id: Pleroma.ScheduledActivityWorker, - start: {Pleroma.ScheduledActivityWorker, :start_link, []} - } + Pleroma.Repo, + Pleroma.Config.TransferTask, + Pleroma.Emoji, + Pleroma.Captcha, + Pleroma.FlakeId, + Pleroma.ScheduledActivityWorker ] ++ + cachex_children() ++ hackney_pool_children() ++ [ - %{ - id: Pleroma.Web.Federator.RetryQueue, - start: {Pleroma.Web.Federator.RetryQueue, :start_link, []} - }, - %{ - id: Pleroma.Web.OAuth.Token.CleanWorker, - start: {Pleroma.Web.OAuth.Token.CleanWorker, :start_link, []} - }, - %{ - id: Pleroma.Stats, - start: {Pleroma.Stats, :start_link, []} - }, + Pleroma.Web.Federator.RetryQueue, + Pleroma.Web.OAuth.Token.CleanWorker, + Pleroma.Stats, %{ id: :web_push_init, start: {Task, :start_link, [&Pleroma.Web.Push.init/0]}, @@ -147,16 +59,11 @@ def start(_type, _args) do restart: :temporary } ] ++ - streamer_child() ++ - chat_child() ++ + streamer_child(@env) ++ + chat_child(@env, chat_enabled?()) ++ [ - # Start the endpoint when the application starts - %{ - id: Pleroma.Web.Endpoint, - start: {Pleroma.Web.Endpoint, :start_link, []}, - type: :supervisor - }, - %{id: Pleroma.Gopher.Server, start: {Pleroma.Gopher.Server, :start_link, []}} + Pleroma.Web.Endpoint, + Pleroma.Gopher.Server ] # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html @@ -201,28 +108,46 @@ def enabled_hackney_pools do end end - if Pleroma.Config.get(:env) == :test do - defp streamer_child, do: [] - defp chat_child, do: [] - else - defp streamer_child do - [%{id: Pleroma.Web.Streamer, start: {Pleroma.Web.Streamer, :start_link, []}}] - end - - defp chat_child do - if Pleroma.Config.get([:chat, :enabled]) do - [ - %{ - id: Pleroma.Web.ChatChannel.ChatChannelState, - start: {Pleroma.Web.ChatChannel.ChatChannelState, :start_link, []} - } - ] - else - [] - end - end + defp cachex_children do + [ + build_cachex("used_captcha", ttl_interval: seconds_valid_interval()), + build_cachex("user", default_ttl: 25_000, ttl_interval: 1000, limit: 2500), + build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500), + build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000), + build_cachex("scrubber", limit: 2500), + build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500) + ] end + defp idempotency_expiration, + do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60)) + + defp seconds_valid_interval, + do: :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid])) + + defp build_cachex(type, opts), + do: %{ + id: String.to_atom("cachex_" <> type), + start: {Cachex, :start_link, [String.to_atom(type <> "_cache"), opts]}, + type: :worker + } + + defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled]) + + defp streamer_child(:test), do: [] + + defp streamer_child(_) do + [Pleroma.Web.Streamer] + end + + defp chat_child(:test, _), do: [] + + defp chat_child(_env, true) do + [Pleroma.Web.ChatChannel.ChatChannelState] + end + + defp chat_child(_, _), do: [] + defp hackney_pool_children do for pool <- enabled_hackney_pools() do options = Pleroma.Config.get([:hackney_pools, pool]) diff --git a/lib/pleroma/captcha/captcha.ex b/lib/pleroma/captcha/captcha.ex index a73b87251..c2765a5b8 100644 --- a/lib/pleroma/captcha/captcha.ex +++ b/lib/pleroma/captcha/captcha.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Captcha do use GenServer @doc false - def start_link do + def start_link(_) do GenServer.start_link(__MODULE__, [], name: __MODULE__) end diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 7799b2a78..3214c9951 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Config.TransferTask do use Task alias Pleroma.Web.AdminAPI.Config - def start_link do + def start_link(_) do load_and_update_env() if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Pleroma.Repo) :ignore diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 052501642..66e20f0e4 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -24,7 +24,7 @@ defmodule Pleroma.Emoji do @ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}] @doc false - def start_link do + def start_link(_) do GenServer.start_link(__MODULE__, [], name: __MODULE__) end diff --git a/lib/pleroma/flake_id.ex b/lib/pleroma/flake_id.ex index ca0610abc..47d61ca5f 100644 --- a/lib/pleroma/flake_id.ex +++ b/lib/pleroma/flake_id.ex @@ -98,7 +98,7 @@ def dump(value) do def autogenerate, do: get() # -- GenServer API - def start_link do + def start_link(_) do :gen_server.start_link({:local, :flake}, __MODULE__, [], []) end diff --git a/lib/pleroma/gopher/server.ex b/lib/pleroma/gopher/server.ex index b3319e137..d4e4f3e55 100644 --- a/lib/pleroma/gopher/server.ex +++ b/lib/pleroma/gopher/server.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Gopher.Server do use GenServer require Logger - def start_link do + def start_link(_) do config = Pleroma.Config.get(:gopher, []) ip = Keyword.get(config, :ip, {0, 0, 0, 0}) port = Keyword.get(config, :port, 1234) diff --git a/lib/pleroma/scheduled_activity_worker.ex b/lib/pleroma/scheduled_activity_worker.ex index 65b38622f..8578cab5e 100644 --- a/lib/pleroma/scheduled_activity_worker.ex +++ b/lib/pleroma/scheduled_activity_worker.ex @@ -16,7 +16,7 @@ defmodule Pleroma.ScheduledActivityWorker do @schedule_interval :timer.minutes(1) - def start_link do + def start_link(_) do GenServer.start_link(__MODULE__, nil) end diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 5b242927b..101effbe4 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -7,7 +7,9 @@ defmodule Pleroma.Stats do alias Pleroma.Repo alias Pleroma.User - def start_link do + use Agent + + def start_link(_) do agent = Agent.start_link(fn -> {[], %{}} end, name: __MODULE__) spawn(fn -> schedule_update() end) agent diff --git a/lib/pleroma/web/chat_channel.ex b/lib/pleroma/web/chat_channel.ex index f63f4bda1..b543909f1 100644 --- a/lib/pleroma/web/chat_channel.ex +++ b/lib/pleroma/web/chat_channel.ex @@ -33,9 +33,11 @@ def handle_in("new_msg", %{"text" => text}, %{assigns: %{user_name: user_name}} end defmodule Pleroma.Web.ChatChannel.ChatChannelState do + use Agent + @max_messages 20 - def start_link do + def start_link(_) do Agent.start_link(fn -> %{max_id: 1, messages: []} end, name: __MODULE__) end diff --git a/lib/pleroma/web/federator/retry_queue.ex b/lib/pleroma/web/federator/retry_queue.ex index 3db948c2e..9eab8c218 100644 --- a/lib/pleroma/web/federator/retry_queue.ex +++ b/lib/pleroma/web/federator/retry_queue.ex @@ -13,7 +13,7 @@ def init(args) do {:ok, %{args | queue_table: queue_table, running_jobs: :sets.new()}} end - def start_link do + def start_link(_) do enabled = if Pleroma.Config.get(:env) == :test, do: true, diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex index dca852449..e39a4986a 100644 --- a/lib/pleroma/web/oauth/token/clean_worker.ex +++ b/lib/pleroma/web/oauth/token/clean_worker.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.OAuth.Token.CleanWorker do @moduledoc """ The module represents functions to clean an expired oauth tokens. """ + use GenServer # 10 seconds @start_interval 10_000 @@ -18,7 +19,7 @@ defmodule Pleroma.Web.OAuth.Token.CleanWorker do alias Pleroma.Web.OAuth.Token - def start_link, do: GenServer.start_link(__MODULE__, nil) + def start_link(_), do: GenServer.start_link(__MODULE__, nil) def init(_) do if Pleroma.Config.get([:oauth2, :clean_expired_tokens], false) do diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 9ee331030..e66378ceb 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -18,7 +18,7 @@ defmodule Pleroma.Web.Streamer do @keepalive_interval :timer.seconds(30) - def start_link do + def start_link(_) do GenServer.start_link(__MODULE__, %{}, name: __MODULE__) end diff --git a/test/config/transfer_task_test.exs b/test/config/transfer_task_test.exs index dbeadbe87..4455a4d47 100644 --- a/test/config/transfer_task_test.exs +++ b/test/config/transfer_task_test.exs @@ -31,7 +31,7 @@ test "transfer config values from db to env" do value: [live: 15, com: 35] }) - Pleroma.Config.TransferTask.start_link() + Pleroma.Config.TransferTask.start_link([]) assert Application.get_env(:pleroma, :test_key) == [live: 2, com: 3] assert Application.get_env(:idna, :test_key) == [live: 15, com: 35] @@ -50,7 +50,7 @@ test "non existing atom" do }) assert ExUnit.CaptureLog.capture_log(fn -> - Pleroma.Config.TransferTask.start_link() + Pleroma.Config.TransferTask.start_link([]) end) =~ "updating env causes error, key: \"undefined_atom_key\", error: %ArgumentError{message: \"argument error\"}" end From 15ef521009f4c232f417ca9164c6be3f4ee4e018 Mon Sep 17 00:00:00 2001 From: stwf Date: Wed, 14 Aug 2019 11:57:50 -0400 Subject: [PATCH 2/5] Isolate OAuth.Token.CleanWorker --- lib/pleroma/application.ex | 9 ++++++++ lib/pleroma/web/oauth/token/clean_worker.ex | 25 ++++++++------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 3bb0718e4..c460a3bc5 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -59,6 +59,7 @@ def start(_type, _args) do restart: :temporary } ] ++ + oauth_cleanup_child(oauth_cleanup_enabled?()) ++ streamer_child(@env) ++ chat_child(@env, chat_enabled?()) ++ [ @@ -134,12 +135,20 @@ defp build_cachex(type, opts), defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled]) + defp oauth_cleanup_enabled?, + do: Pleroma.Config.get([:oauth2, :clean_expired_tokens], false) + defp streamer_child(:test), do: [] defp streamer_child(_) do [Pleroma.Web.Streamer] end + defp oauth_cleanup_child(true), + do: [Pleroma.Web.OAuth.Token.CleanWorker] + + defp oauth_cleanup_child(_), do: [] + defp chat_child(:test, _), do: [] defp chat_child(_env, true) do diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex index e39a4986a..f50098302 100644 --- a/lib/pleroma/web/oauth/token/clean_worker.ex +++ b/lib/pleroma/web/oauth/token/clean_worker.ex @@ -8,35 +8,28 @@ defmodule Pleroma.Web.OAuth.Token.CleanWorker do """ use GenServer - # 10 seconds - @start_interval 10_000 + @ten_seconds 10_000 + @one_day 86_400_000 + @interval Pleroma.Config.get( - # 24 hours [:oauth2, :clean_expired_tokens_interval], - 86_400_000 + @one_day ) - @queue :background alias Pleroma.Web.OAuth.Token - def start_link(_), do: GenServer.start_link(__MODULE__, nil) + def start_link(_), do: GenServer.start_link(__MODULE__, %{}) def init(_) do - if Pleroma.Config.get([:oauth2, :clean_expired_tokens], false) do - Process.send_after(self(), :perform, @start_interval) - {:ok, nil} - else - :ignore - end + Process.send_after(self(), :perform, @ten_seconds) + {:ok, nil} end @doc false def handle_info(:perform, state) do + Token.delete_expired_tokens() + Process.send_after(self(), :perform, @interval) - PleromaJobQueue.enqueue(@queue, __MODULE__, [:clean]) {:noreply, state} end - - # Job Worker Callbacks - def perform(:clean), do: Token.delete_expired_tokens() end From 574856ef01fae7ea411ec363929ab9a22d76a65d Mon Sep 17 00:00:00 2001 From: stwf Date: Wed, 14 Aug 2019 11:58:32 -0400 Subject: [PATCH 3/5] streamline Streamer pings --- lib/pleroma/web/streamer.ex | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index e66378ceb..bbaddd852 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -35,28 +35,21 @@ def stream(topic, item) do end def init(args) do - spawn(fn -> - # 30 seconds - Process.sleep(@keepalive_interval) - GenServer.cast(__MODULE__, %{action: :ping}) - end) + Process.send_after(self(), %{action: :ping}, @keepalive_interval) {:ok, args} end - def handle_cast(%{action: :ping}, topics) do - Map.values(topics) + def handle_info(%{action: :ping}, topics) do + topics + |> Map.values() |> List.flatten() |> Enum.each(fn socket -> Logger.debug("Sending keepalive ping") send(socket.transport_pid, {:text, ""}) end) - spawn(fn -> - # 30 seconds - Process.sleep(@keepalive_interval) - GenServer.cast(__MODULE__, %{action: :ping}) - end) + Process.send_after(self(), %{action: :ping}, @keepalive_interval) {:noreply, topics} end From d81f63845a71e5cc60d95007ecaa2aea52a90422 Mon Sep 17 00:00:00 2001 From: stwf Date: Wed, 14 Aug 2019 11:59:33 -0400 Subject: [PATCH 4/5] Implement Pleroma.Stats as GenServer --- lib/pleroma/stats.ex | 60 +++++++++++++------ .../mastodon_api_controller_test.exs | 4 +- 2 files changed, 44 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 101effbe4..a3b8a4d66 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -7,33 +7,56 @@ defmodule Pleroma.Stats do alias Pleroma.Repo alias Pleroma.User - use Agent + use GenServer + + @interval 1000 * 60 * 60 def start_link(_) do - agent = Agent.start_link(fn -> {[], %{}} end, name: __MODULE__) - spawn(fn -> schedule_update() end) - agent + GenServer.start_link(__MODULE__, initial_data(), name: __MODULE__) + end + + def force_update do + GenServer.call(__MODULE__, :force_update) end def get_stats do - Agent.get(__MODULE__, fn {_, stats} -> stats end) + %{stats: stats} = GenServer.call(__MODULE__, :get_state) + + stats end def get_peers do - Agent.get(__MODULE__, fn {peers, _} -> peers end) + %{peers: peers} = GenServer.call(__MODULE__, :get_state) + + peers end - def schedule_update do - spawn(fn -> - # 1 hour - Process.sleep(1000 * 60 * 60) - schedule_update() - end) - - update_stats() + def init(args) do + Process.send_after(self(), :run_update, @interval) + {:ok, args} end - def update_stats do + def handle_call(:force_update, _from, _state) do + new_stats = get_stat_data() + {:reply, new_stats, new_stats} + end + + def handle_call(:get_state, _from, state) do + {:reply, state, state} + end + + def handle_info(:run_update, _state) do + new_stats = get_stat_data() + + Process.send_after(self(), :run_update, @interval) + {:noreply, new_stats} + end + + defp initial_data do + %{peers: [], stats: %{}} + end + + defp get_stat_data do peers = from( u in User, @@ -54,8 +77,9 @@ def update_stats do user_count = Repo.aggregate(User.Query.build(%{local: true, active: true}), :count, :id) - Agent.update(__MODULE__, fn _ -> - {peers, %{domain_count: domain_count, status_count: status_count, user_count: user_count}} - end) + %{ + peers: peers, + stats: %{domain_count: domain_count, status_count: status_count, user_count: user_count} + } end end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 2febe8b3a..112e272f9 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2624,7 +2624,7 @@ test "get instance stats", %{conn: conn} do |> Changeset.put_embed(:info, info_change) |> User.update_and_set_cache() - Pleroma.Stats.update_stats() + Pleroma.Stats.force_update() conn = get(conn, "/api/v1/instance") @@ -2642,7 +2642,7 @@ test "get peers", %{conn: conn} do insert(:user, %{local: false, nickname: "u@peer1.com"}) insert(:user, %{local: false, nickname: "u@peer2.com"}) - Pleroma.Stats.update_stats() + Pleroma.Stats.force_update() conn = get(conn, "/api/v1/instance/peers") From c43152f6c17a287a9fe4f2556ca20a140ea30248 Mon Sep 17 00:00:00 2001 From: stwf Date: Wed, 14 Aug 2019 14:01:11 -0400 Subject: [PATCH 5/5] fix formatting --- lib/pleroma/application.ex | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index c460a3bc5..aa673188f 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -121,17 +121,17 @@ defp cachex_children do end defp idempotency_expiration, - do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60)) + do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60)) defp seconds_valid_interval, - do: :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid])) + do: :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid])) defp build_cachex(type, opts), - do: %{ - id: String.to_atom("cachex_" <> type), - start: {Cachex, :start_link, [String.to_atom(type <> "_cache"), opts]}, - type: :worker - } + do: %{ + id: String.to_atom("cachex_" <> type), + start: {Cachex, :start_link, [String.to_atom(type <> "_cache"), opts]}, + type: :worker + } defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled]) @@ -145,7 +145,7 @@ defp streamer_child(_) do end defp oauth_cleanup_child(true), - do: [Pleroma.Web.OAuth.Token.CleanWorker] + do: [Pleroma.Web.OAuth.Token.CleanWorker] defp oauth_cleanup_child(_), do: []