removing Stats worker from Oban cron jobs

This commit is contained in:
Alexander Strizhakov 2020-09-06 12:13:26 +03:00
parent 6c6de8e5df
commit ee67c98e55
No known key found for this signature in database
GPG key ID: 022896A53AEF1381
10 changed files with 127 additions and 50 deletions

View file

@ -6,9 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
## unreleased-patch - ??? ## unreleased-patch - ???
### Added ### Added
- Rich media failure tracking (along with `:failure_backoff` option) - Rich media failure tracking (along with `:failure_backoff` option)
### Fixed ### Fixed
- Possible OOM errors with the default HTTP adapter - Possible OOM errors with the default HTTP adapter
- Mastodon API: Search parameter `following` now correctly returns the followings rather than the followers - Mastodon API: Search parameter `following` now correctly returns the followings rather than the followers
- Mastodon API: Timelines hanging for (`number of posts with links * rich media timeout`) in the worst case. - Mastodon API: Timelines hanging for (`number of posts with links * rich media timeout`) in the worst case.
@ -16,6 +18,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Mastodon API: Cards being wrong for preview statuses due to cache key collision - Mastodon API: Cards being wrong for preview statuses due to cache key collision
- Password resets no longer processed for deactivated accounts - Password resets no longer processed for deactivated accounts
## Unreleased
### Removed
- **Breaking:** Removed `Pleroma.Workers.Cron.StatsWorker` setting from Oban `:crontab`.
## [2.1.0] - 2020-08-28 ## [2.1.0] - 2020-08-28
### Changed ### Changed

View file

@ -546,7 +546,6 @@ config :pleroma, Oban,
plugins: [Oban.Plugins.Pruner], plugins: [Oban.Plugins.Pruner],
crontab: [ crontab: [
{"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker},
{"0 * * * *", Pleroma.Workers.Cron.StatsWorker},
{"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker}, {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker},
{"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker}, {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker},
{"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker} {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker}

View file

@ -2291,7 +2291,6 @@ config :pleroma, :config_description, [
description: "Settings for cron background jobs", description: "Settings for cron background jobs",
suggestions: [ suggestions: [
{"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker},
{"0 * * * *", Pleroma.Workers.Cron.StatsWorker},
{"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker}, {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker},
{"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker}, {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker},
{"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker} {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker}

View file

@ -18,6 +18,7 @@ defmodule Mix.Pleroma do
@doc "Common functions to be reused in mix tasks" @doc "Common functions to be reused in mix tasks"
def start_pleroma do def start_pleroma do
Pleroma.Config.Holder.save_default() Pleroma.Config.Holder.save_default()
Pleroma.Config.Oban.warn()
Application.put_env(:phoenix, :serve_endpoints, false, persistent: true) Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
if Pleroma.Config.get(:env) != :test do if Pleroma.Config.get(:env) != :test do

View file

@ -50,6 +50,7 @@ defmodule Pleroma.Application do
Pleroma.Telemetry.Logger.attach() Pleroma.Telemetry.Logger.attach()
Config.Holder.save_default() Config.Holder.save_default()
Pleroma.HTML.compile_scrubbers() Pleroma.HTML.compile_scrubbers()
Pleroma.Config.Oban.warn()
Config.DeprecationWarnings.warn() Config.DeprecationWarnings.warn()
Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled() Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
Pleroma.ApplicationRequirements.verify!() Pleroma.ApplicationRequirements.verify!()

View file

@ -0,0 +1,30 @@
defmodule Pleroma.Config.Oban do
require Logger
def warn do
oban_config = Pleroma.Config.get(Oban)
crontab =
[Pleroma.Workers.Cron.StatsWorker]
|> Enum.reduce(oban_config[:crontab], fn removed_worker, acc ->
with acc when is_list(acc) <- acc,
setting when is_tuple(setting) <-
Enum.find(acc, fn {_, worker} -> worker == removed_worker end) do
"""
!!!OBAN CONFIG WARNING!!!
You are using old workers in Oban crontab settings, which were removed.
Please, remove setting from crontab in your config file (prod.secret.exs): #{
inspect(setting)
}
"""
|> Logger.warn()
List.delete(acc, setting)
else
_ -> acc
end
end)
Pleroma.Config.put(Oban, Keyword.put(oban_config, :crontab, crontab))
end
end

View file

@ -3,12 +3,15 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Stats do defmodule Pleroma.Stats do
use GenServer
import Ecto.Query import Ecto.Query
alias Pleroma.CounterCache alias Pleroma.CounterCache
alias Pleroma.Repo alias Pleroma.Repo
alias Pleroma.User alias Pleroma.User
use GenServer @interval :timer.seconds(60)
def start_link(_) do def start_link(_) do
GenServer.start_link( GenServer.start_link(
@ -18,6 +21,11 @@ defmodule Pleroma.Stats do
) )
end end
@impl true
def init(_args) do
{:ok, nil, {:continue, :calculate_stats}}
end
@doc "Performs update stats" @doc "Performs update stats"
def force_update do def force_update do
GenServer.call(__MODULE__, :force_update) GenServer.call(__MODULE__, :force_update)
@ -29,7 +37,11 @@ defmodule Pleroma.Stats do
end end
@doc "Returns stats data" @doc "Returns stats data"
@spec get_stats() :: %{domain_count: integer(), status_count: integer(), user_count: integer()} @spec get_stats() :: %{
domain_count: non_neg_integer(),
status_count: non_neg_integer(),
user_count: non_neg_integer()
}
def get_stats do def get_stats do
%{stats: stats} = GenServer.call(__MODULE__, :get_state) %{stats: stats} = GenServer.call(__MODULE__, :get_state)
@ -44,25 +56,14 @@ defmodule Pleroma.Stats do
peers peers
end end
def init(_args) do @spec calculate_stat_data() :: %{
{:ok, calculate_stat_data()} peers: list(),
end stats: %{
domain_count: non_neg_integer(),
def handle_call(:force_update, _from, _state) do status_count: non_neg_integer(),
new_stats = calculate_stat_data() user_count: non_neg_integer()
{:reply, new_stats, new_stats} }
end }
def handle_call(:get_state, _from, state) do
{:reply, state, state}
end
def handle_cast(:run_update, _state) do
new_stats = calculate_stat_data()
{:noreply, new_stats}
end
def calculate_stat_data do def calculate_stat_data do
peers = peers =
from( from(
@ -97,6 +98,7 @@ defmodule Pleroma.Stats do
} }
end end
@spec get_status_visibility_count(String.t() | nil) :: map()
def get_status_visibility_count(instance \\ nil) do def get_status_visibility_count(instance \\ nil) do
if is_nil(instance) do if is_nil(instance) do
CounterCache.get_sum() CounterCache.get_sum()
@ -104,4 +106,36 @@ defmodule Pleroma.Stats do
CounterCache.get_by_instance(instance) CounterCache.get_by_instance(instance)
end end
end end
@impl true
def handle_continue(:calculate_stats, _) do
stats = calculate_stat_data()
Process.send_after(self(), :run_update, @interval)
{:noreply, stats}
end
@impl true
def handle_call(:force_update, _from, _state) do
new_stats = calculate_stat_data()
{:reply, new_stats, new_stats}
end
@impl true
def handle_call(:get_state, _from, state) do
{:reply, state, state}
end
@impl true
def handle_cast(:run_update, _state) do
new_stats = calculate_stat_data()
{:noreply, new_stats}
end
@impl true
def handle_info(:run_update, _) do
new_stats = calculate_stat_data()
Process.send_after(self(), :run_update, @interval)
{:noreply, new_stats}
end
end end

View file

@ -1,17 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.Cron.StatsWorker do
@moduledoc """
The worker to update peers statistics.
"""
use Oban.Worker, queue: "background"
@impl Oban.Worker
def perform(_job) do
Pleroma.Stats.do_collect()
:ok
end
end

View file

@ -0,0 +1,19 @@
defmodule Pleroma.Repo.Migrations.RemoveCronStatsWorkerFromObanConfig do
use Ecto.Migration
def change do
with %Pleroma.ConfigDB{} = config <-
Pleroma.ConfigDB.get_by_params(%{group: :pleroma, key: Oban}),
crontab when is_list(crontab) <- config.value[:crontab],
index when is_integer(index) <-
Enum.find_index(crontab, fn {_, worker} ->
worker == Pleroma.Workers.Cron.StatsWorker
end) do
updated_value = Keyword.put(config.value, :crontab, List.delete_at(crontab, index))
config
|> Ecto.Changeset.change(value: updated_value)
|> Pleroma.Repo.update()
end
end
end

View file

@ -4,7 +4,10 @@
defmodule Pleroma.StatsTest do defmodule Pleroma.StatsTest do
use Pleroma.DataCase use Pleroma.DataCase
import Pleroma.Factory import Pleroma.Factory
alias Pleroma.Stats
alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI
describe "user count" do describe "user count" do
@ -13,7 +16,7 @@ defmodule Pleroma.StatsTest do
_internal = insert(:user, local: true, nickname: nil) _internal = insert(:user, local: true, nickname: nil)
_internal = Pleroma.Web.ActivityPub.Relay.get_actor() _internal = Pleroma.Web.ActivityPub.Relay.get_actor()
assert match?(%{stats: %{user_count: 1}}, Pleroma.Stats.calculate_stat_data()) assert match?(%{stats: %{user_count: 1}}, Stats.calculate_stat_data())
end end
end end
@ -47,23 +50,23 @@ defmodule Pleroma.StatsTest do
end) end)
assert %{"direct" => 3, "private" => 4, "public" => 1, "unlisted" => 2} = assert %{"direct" => 3, "private" => 4, "public" => 1, "unlisted" => 2} =
Pleroma.Stats.get_status_visibility_count() Stats.get_status_visibility_count()
end end
test "on status delete" do test "on status delete" do
user = insert(:user) user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"}) {:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"})
assert %{"public" => 1} = Pleroma.Stats.get_status_visibility_count() assert %{"public" => 1} = Stats.get_status_visibility_count()
CommonAPI.delete(activity.id, user) CommonAPI.delete(activity.id, user)
assert %{"public" => 0} = Pleroma.Stats.get_status_visibility_count() assert %{"public" => 0} = Stats.get_status_visibility_count()
end end
test "on status visibility update" do test "on status visibility update" do
user = insert(:user) user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"}) {:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"})
assert %{"public" => 1, "private" => 0} = Pleroma.Stats.get_status_visibility_count() assert %{"public" => 1, "private" => 0} = Stats.get_status_visibility_count()
{:ok, _} = CommonAPI.update_activity_scope(activity.id, %{visibility: "private"}) {:ok, _} = CommonAPI.update_activity_scope(activity.id, %{visibility: "private"})
assert %{"public" => 0, "private" => 1} = Pleroma.Stats.get_status_visibility_count() assert %{"public" => 0, "private" => 1} = Stats.get_status_visibility_count()
end end
test "doesn't count unrelated activities" do test "doesn't count unrelated activities" do
@ -75,7 +78,7 @@ defmodule Pleroma.StatsTest do
CommonAPI.repeat(activity.id, other_user) CommonAPI.repeat(activity.id, other_user)
assert %{"direct" => 0, "private" => 0, "public" => 1, "unlisted" => 0} = assert %{"direct" => 0, "private" => 0, "public" => 1, "unlisted" => 0} =
Pleroma.Stats.get_status_visibility_count() Stats.get_status_visibility_count()
end end
end end
@ -110,10 +113,10 @@ defmodule Pleroma.StatsTest do
end) end)
assert %{"direct" => 10, "private" => 0, "public" => 1, "unlisted" => 5} = assert %{"direct" => 10, "private" => 0, "public" => 1, "unlisted" => 5} =
Pleroma.Stats.get_status_visibility_count(local_instance) Stats.get_status_visibility_count(local_instance)
assert %{"direct" => 0, "private" => 20, "public" => 0, "unlisted" => 0} = assert %{"direct" => 0, "private" => 20, "public" => 0, "unlisted" => 0} =
Pleroma.Stats.get_status_visibility_count(instance2) Stats.get_status_visibility_count(instance2)
end end
end end
end end