From 0e9be6bafa81b4a55546b41e8b217fbe559cd16d Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 25 Nov 2019 14:32:20 +0300 Subject: [PATCH 01/19] moved OAuth.Token.CleanWorker to Oban Periodic jobs --- config/config.exs | 6 ++- config/description.exs | 7 ---- docs/configuration/cheatsheet.md | 13 ++++++- lib/pleroma/application.ex | 9 ----- lib/pleroma/web/oauth/token/clean_worker.ex | 38 ------------------- lib/pleroma/workers/background_worker.ex | 5 --- .../workers/cron/clear_oauth_token_worker.ex | 21 ++++++++++ mix.exs | 2 +- mix.lock | 8 ++-- 9 files changed, 41 insertions(+), 68 deletions(-) delete mode 100644 lib/pleroma/web/oauth/token/clean_worker.ex create mode 100644 lib/pleroma/workers/cron/clear_oauth_token_worker.ex diff --git a/config/config.exs b/config/config.exs index 1e36d3314..79d4ee55f 100644 --- a/config/config.exs +++ b/config/config.exs @@ -505,6 +505,9 @@ transmogrifier: 20, scheduled_activities: 10, background: 5 + ], + crontab: [ + {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker} ] config :pleroma, :workers, @@ -594,8 +597,7 @@ config :pleroma, :oauth2, token_expires_in: 600, issue_new_refresh_token: true, - clean_expired_tokens: false, - clean_expired_tokens_interval: 86_400_000 + clean_expired_tokens: false config :pleroma, :database, rum_enabled: false diff --git a/config/description.exs b/config/description.exs index 70e963399..6b89d7c67 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2251,13 +2251,6 @@ key: :clean_expired_tokens, type: :boolean, description: "Enable a background job to clean expired oauth tokens. Defaults to false" - }, - %{ - key: :clean_expired_tokens_interval, - type: :integer, - description: - "Interval to run the job to clean expired tokens. Defaults to 86_400_000 (24 hours).", - suggestions: [86_400_000] } ] }, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 07d9a1d45..e414d0b11 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -483,6 +483,7 @@ Configuration options described in [Oban readme](https://github.com/sorentwo/oba * `verbose` - logs verbosity * `prune` - non-retryable jobs [pruning settings](https://github.com/sorentwo/oban#pruning) (`:disabled` / `{:maxlen, value}` / `{:maxage, value}`) * `queues` - job queues (see below) +* `crontab` - periodic jobs, see [`Oban.Cron`](#obancron) Pleroma has the following queues: @@ -494,6 +495,12 @@ Pleroma has the following queues: * `web_push` - Web push notifications * `scheduled_activities` - Scheduled activities, see [`Pleroma.ScheduledActivity`](#pleromascheduledactivity) +#### Oban.Cron + +Pleroma has the periodic jobs: + +`Pleroma.Workers.Cron.ClearOauthTokenWorker` - the job to clean an expired oauth tokens. + Example: ```elixir @@ -504,6 +511,9 @@ config :pleroma, Oban, queues: [ federator_incoming: 50, federator_outgoing: 50 + ], + crontab: [ + {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker} ] ``` @@ -786,8 +796,7 @@ Configure OAuth 2 provider capabilities: * `token_expires_in` - The lifetime in seconds of the access token. * `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token. -* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`. -* `clean_expired_tokens_interval` - Interval to run the job to clean expired tokens. Defaults to `86_400_000` (24 hours). +* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`. Interval settings sets in configuration periodic jobs [`Oban.Cron`](#obancron) ## Link parsing diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 2b6a55f98..69a1e4a50 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -47,7 +47,6 @@ def start(_type, _args) do {Oban, Pleroma.Config.get(Oban)} ] ++ task_children(@env) ++ - oauth_cleanup_child(oauth_cleanup_enabled?()) ++ streamer_child(@env) ++ chat_child(@env, chat_enabled?()) ++ [ @@ -127,20 +126,12 @@ defp build_cachex(type, opts), defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled]) - defp oauth_cleanup_enabled?, - do: Pleroma.Config.get([:oauth2, :clean_expired_tokens], false) - defp streamer_child(:test), do: [] defp streamer_child(_) do [Pleroma.Web.Streamer.supervisor()] end - defp oauth_cleanup_child(true), - do: [Pleroma.Web.OAuth.Token.CleanWorker] - - defp oauth_cleanup_child(_), do: [] - defp chat_child(:test, _), do: [] defp chat_child(_env, true) do diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex deleted file mode 100644 index f639f9c6f..000000000 --- a/lib/pleroma/web/oauth/token/clean_worker.ex +++ /dev/null @@ -1,38 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OAuth.Token.CleanWorker do - @moduledoc """ - The module represents functions to clean an expired oauth tokens. - """ - use GenServer - - @ten_seconds 10_000 - @one_day 86_400_000 - - @interval Pleroma.Config.get( - [:oauth2, :clean_expired_tokens_interval], - @one_day - ) - - alias Pleroma.Web.OAuth.Token - alias Pleroma.Workers.BackgroundWorker - - def start_link(_), do: GenServer.start_link(__MODULE__, %{}) - - def init(_) do - Process.send_after(self(), :perform, @ten_seconds) - {:ok, nil} - end - - @doc false - def handle_info(:perform, state) do - BackgroundWorker.enqueue("clean_expired_tokens", %{}) - - Process.send_after(self(), :perform, @interval) - {:noreply, state} - end - - def perform(:clean), do: Token.delete_expired_tokens() -end diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index 7ffc8eabe..bb2b37b18 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Workers.BackgroundWorker do alias Pleroma.Activity alias Pleroma.User alias Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy - alias Pleroma.Web.OAuth.Token.CleanWorker use Pleroma.Workers.WorkerHelper, queue: "background" @@ -55,10 +54,6 @@ def perform( User.perform(:follow_import, follower, followed_identifiers) end - def perform(%{"op" => "clean_expired_tokens"}, _job) do - CleanWorker.perform(:clean) - end - def perform(%{"op" => "media_proxy_preload", "message" => message}, _job) do MediaProxyWarmingPolicy.perform(:preload, message) end diff --git a/lib/pleroma/workers/cron/clear_oauth_token_worker.ex b/lib/pleroma/workers/cron/clear_oauth_token_worker.ex new file mode 100644 index 000000000..1a4cdc198 --- /dev/null +++ b/lib/pleroma/workers/cron/clear_oauth_token_worker.ex @@ -0,0 +1,21 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.ClearOauthTokenWorker do + @moduledoc """ + The worker to clean an expired oauth tokens. + """ + + use Oban.Worker, queue: "background" + + alias Pleroma.Config + alias Pleroma.Web.OAuth.Token + + @impl Oban.Worker + def perform(_opts, _job) do + if Config.get([:oauth2, :clean_expired_tokens], false) do + Token.delete_expired_tokens() + end + end +end diff --git a/mix.exs b/mix.exs index 60c7fe3f6..31deae96e 100644 --- a/mix.exs +++ b/mix.exs @@ -102,7 +102,7 @@ defp deps do {:phoenix_ecto, "~> 4.0"}, {:ecto_sql, "~> 3.2"}, {:postgrex, ">= 0.13.5"}, - {:oban, "~> 0.8.1"}, + {:oban, "~> 0.11.1"}, {:quantum, "~> 2.3"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, diff --git a/mix.lock b/mix.lock index 3a664287c..c212b39e0 100644 --- a/mix.lock +++ b/mix.lock @@ -23,8 +23,8 @@ "decimal": {:hex, :decimal, "1.8.0", "ca462e0d885f09a1c5a342dbd7c1dcf27ea63548c65a65e67334f4b61803822e", [:mix], [], "hexpm"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, "earmark": {:hex, :earmark, "1.4.2", "3aa0bd23bc4c61cf2f1e5d752d1bb470560a6f8539974f767a38923bb20e1d7f", [:mix], [], "hexpm"}, - "ecto": {:hex, :ecto, "3.2.3", "51274df79862845b388733fddcf6f107d0c8c86e27abe7131fa98f8d30761bda", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, - "ecto_sql": {:hex, :ecto_sql, "3.2.0", "751cea597e8deb616084894dd75cbabfdbe7255ff01e8c058ca13f0353a3921b", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.2.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "ecto": {:hex, :ecto, "3.2.5", "76c864b77948a479e18e69cc1d0f0f4ee7cced1148ffe6a093ff91eba644f0b5", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "ecto_sql": {:hex, :ecto_sql, "3.2.1", "4eed4100cbb2abcff10c46660d6613693807bf64f1b865f414daccf762d3758d", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.2.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"}, "eternal": {:hex, :eternal, "1.2.1", "d5b6b2499ba876c57be2581b5b999ee9bdf861c647401066d3eeed111d096bc4", [:mix], [], "hexpm"}, "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"}, @@ -67,7 +67,7 @@ "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "ad0097e2f61d4953bfef20fb6abddf23b87111e6", [ref: "ad0097e2f61d4953bfef20fb6abddf23b87111e6", submodules: true]}, "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"}, "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]}, - "oban": {:hex, :oban, "0.8.1", "4bbf62eb1829f856d69aeb5069ac7036afe07db8221a17de2a9169cc7a58a318", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "oban": {:hex, :oban, "0.11.1", "e34964fad7f188c2c3d006485601a8897e537f7b88a31928be2833ae1cab59af", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.4", "8dd29ed783f2e12195d7e0a4640effc0a7c37e6537da491f1db01839eee6d053", [:mix], [], "hexpm"}, "phoenix": {:hex, :phoenix, "1.4.10", "619e4a545505f562cd294df52294372d012823f4fd9d34a6657a8b242898c255", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, @@ -97,7 +97,7 @@ "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.23.5", "bfd9404bbf5069b1be2ffd317923ce57e58b332e25dbca2a35dedd7820dfee5a", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, - "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, + "telemetry": {:hex, :telemetry, "0.4.1", "ae2718484892448a24470e6aa341bc847c3277bfb8d4e9289f7474d752c09c7f", [:rebar3], [], "hexpm"}, "tesla": {:hex, :tesla, "1.3.0", "f35d72f029e608f9cdc6f6d6fcc7c66cf6d6512a70cfef9206b21b8bd0203a30", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 0.4", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm"}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, From ac3abb5414bd7a5bbf53678cdf02b6f59063124c Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 26 Nov 2019 10:53:07 +0300 Subject: [PATCH 02/19] moved Pleroma.Stats to Oban Periodic jobs --- config/config.exs | 3 +- lib/pleroma/stats.ex | 39 ++++++++++++++++++------ lib/pleroma/workers/cron/stats_worker.ex | 16 ++++++++++ test/web/node_info_test.exs | 1 + 4 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 lib/pleroma/workers/cron/stats_worker.ex diff --git a/config/config.exs b/config/config.exs index 79d4ee55f..5fc92ca1b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -507,7 +507,8 @@ background: 5 ], crontab: [ - {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker} + {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, + {"0 * * * *", Pleroma.Workers.Cron.StatsWorker} ] config :pleroma, :workers, diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 8154a09b7..cf590fb01 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -9,22 +9,43 @@ defmodule Pleroma.Stats do use GenServer - @interval 1000 * 60 * 60 + @init_state %{ + peers: [], + stats: %{ + domain_count: 0, + status_count: 0, + user_count: 0 + } + } def start_link(_) do - GenServer.start_link(__MODULE__, initial_data(), name: __MODULE__) + GenServer.start_link( + __MODULE__, + @init_state, + name: __MODULE__ + ) end + @doc "Performs update stats" def force_update do GenServer.call(__MODULE__, :force_update) end + @doc "Performs collect stats" + def do_collect do + GenServer.cast(__MODULE__, :run_update) + end + + @doc "Returns stats data" + @spec get_stats() :: %{domain_count: integer(), status_count: integer(), user_count: integer()} def get_stats do %{stats: stats} = GenServer.call(__MODULE__, :get_state) stats end + @doc "Returns list peers" + @spec get_peers() :: list(String.t()) def get_peers do %{peers: peers} = GenServer.call(__MODULE__, :get_state) @@ -32,7 +53,6 @@ def get_peers do end def init(args) do - Process.send(self(), :run_update, []) {:ok, args} end @@ -45,17 +65,12 @@ def handle_call(:get_state, _from, state) do {:reply, state, state} end - def handle_info(:run_update, _state) do + def handle_cast(:run_update, _state) do new_stats = get_stat_data() - Process.send_after(self(), :run_update, @interval) {:noreply, new_stats} end - defp initial_data do - %{peers: [], stats: %{}} - end - defp get_stat_data do peers = from( @@ -74,7 +89,11 @@ defp get_stat_data do %{ peers: peers, - stats: %{domain_count: domain_count, status_count: status_count, user_count: user_count} + stats: %{ + domain_count: domain_count, + status_count: status_count, + user_count: user_count + } } end end diff --git a/lib/pleroma/workers/cron/stats_worker.ex b/lib/pleroma/workers/cron/stats_worker.ex new file mode 100644 index 000000000..425ad41ca --- /dev/null +++ b/lib/pleroma/workers/cron/stats_worker.ex @@ -0,0 +1,16 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.StatsWorker do + @moduledoc """ + The worker to update peers statistics. + """ + + use Oban.Worker, queue: "background" + + @impl Oban.Worker + def perform(_opts, _job) do + Pleroma.Stats.do_collect() + end +end diff --git a/test/web/node_info_test.exs b/test/web/node_info_test.exs index 9a574a38d..39dd72cec 100644 --- a/test/web/node_info_test.exs +++ b/test/web/node_info_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.NodeInfoTest do use Pleroma.Web.ConnCase import Pleroma.Factory + clear_config([:mrf_simple]) test "GET /.well-known/nodeinfo", %{conn: conn} do links = From 6f202a401babbc73e93fb09e697c4c844688f557 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 27 Nov 2019 09:26:37 +0300 Subject: [PATCH 03/19] moved ScheduledActivity to Oban Periodic jobs --- config/config.exs | 3 +- lib/pleroma/application.ex | 1 - .../cron/scheduled_activity_worker.ex} | 41 +++++-------------- .../workers/scheduled_activity_worker.ex | 12 ------ .../scheduled_activity_daemon_test.exs | 19 --------- test/scheduled_activity_test.exs | 28 +++++++++++++ 6 files changed, 40 insertions(+), 64 deletions(-) rename lib/pleroma/{daemons/scheduled_activity_daemon.ex => workers/cron/scheduled_activity_worker.ex} (51%) delete mode 100644 lib/pleroma/workers/scheduled_activity_worker.ex delete mode 100644 test/daemons/scheduled_activity_daemon_test.exs diff --git a/config/config.exs b/config/config.exs index 5fc92ca1b..f4f7fcce7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -508,7 +508,8 @@ ], crontab: [ {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, - {"0 * * * *", Pleroma.Workers.Cron.StatsWorker} + {"0 * * * *", Pleroma.Workers.Cron.StatsWorker}, + {"* * * * *", Pleroma.Workers.Cron.ScheduledActivityWorker} ] config :pleroma, :workers, diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 69a1e4a50..71670d9fc 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -35,7 +35,6 @@ def start(_type, _args) do Pleroma.Config.TransferTask, Pleroma.Emoji, Pleroma.Captcha, - Pleroma.Daemons.ScheduledActivityDaemon, Pleroma.Daemons.ActivityExpirationDaemon, Pleroma.Plugs.RateLimiter.Supervisor ] ++ diff --git a/lib/pleroma/daemons/scheduled_activity_daemon.ex b/lib/pleroma/workers/cron/scheduled_activity_worker.ex similarity index 51% rename from lib/pleroma/daemons/scheduled_activity_daemon.ex rename to lib/pleroma/workers/cron/scheduled_activity_worker.ex index aee5f723a..407ab687a 100644 --- a/lib/pleroma/daemons/scheduled_activity_daemon.ex +++ b/lib/pleroma/workers/cron/scheduled_activity_worker.ex @@ -2,37 +2,33 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Daemons.ScheduledActivityDaemon do +defmodule Pleroma.Workers.Cron.ScheduledActivityWorker do @moduledoc """ - Sends scheduled activities to the job queue. + The worker to post scheduled actvities. """ + use Oban.Worker, queue: "scheduled_activities" alias Pleroma.Config alias Pleroma.ScheduledActivity alias Pleroma.User alias Pleroma.Web.CommonAPI - use GenServer require Logger @schedule_interval :timer.minutes(1) - def start_link(_) do - GenServer.start_link(__MODULE__, nil) - end - - def init(_) do + @impl Oban.Worker + def perform(_opts, _job) do if Config.get([ScheduledActivity, :enabled]) do - schedule_next() - {:ok, nil} - else - :ignore + @schedule_interval + |> ScheduledActivity.due_activities() + |> Enum.each(&post_activity/1) end end - def perform(:execute, scheduled_activity_id) do + def post_activity(scheduled_activity) do try do - {:ok, scheduled_activity} = ScheduledActivity.delete(scheduled_activity_id) + {:ok, scheduled_activity} = ScheduledActivity.delete(scheduled_activity) %User{} = user = User.get_cached_by_id(scheduled_activity.user_id) {:ok, _result} = CommonAPI.post(user, scheduled_activity.params) rescue @@ -42,21 +38,4 @@ def perform(:execute, scheduled_activity_id) do ) end end - - def handle_info(:perform, state) do - ScheduledActivity.due_activities(@schedule_interval) - |> Enum.each(fn scheduled_activity -> - Pleroma.Workers.ScheduledActivityWorker.enqueue( - "execute", - %{"activity_id" => scheduled_activity.id} - ) - end) - - schedule_next() - {:noreply, state} - end - - defp schedule_next do - Process.send_after(self(), :perform, @schedule_interval) - end end diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex deleted file mode 100644 index ca7d53af1..000000000 --- a/lib/pleroma/workers/scheduled_activity_worker.ex +++ /dev/null @@ -1,12 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.ScheduledActivityWorker do - use Pleroma.Workers.WorkerHelper, queue: "scheduled_activities" - - @impl Oban.Worker - def perform(%{"op" => "execute", "activity_id" => activity_id}, _job) do - Pleroma.Daemons.ScheduledActivityDaemon.perform(:execute, activity_id) - end -end diff --git a/test/daemons/scheduled_activity_daemon_test.exs b/test/daemons/scheduled_activity_daemon_test.exs deleted file mode 100644 index c8e464491..000000000 --- a/test/daemons/scheduled_activity_daemon_test.exs +++ /dev/null @@ -1,19 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.ScheduledActivityDaemonTest do - use Pleroma.DataCase - alias Pleroma.ScheduledActivity - import Pleroma.Factory - - test "creates a status from the scheduled activity" do - user = insert(:user) - scheduled_activity = insert(:scheduled_activity, user: user, params: %{status: "hi"}) - Pleroma.Daemons.ScheduledActivityDaemon.perform(:execute, scheduled_activity.id) - - refute Repo.get(ScheduledActivity, scheduled_activity.id) - activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) - assert Pleroma.Object.normalize(activity).data["content"] == "hi" - end -end diff --git a/test/scheduled_activity_test.exs b/test/scheduled_activity_test.exs index dcf12fb49..d3d05745f 100644 --- a/test/scheduled_activity_test.exs +++ b/test/scheduled_activity_test.exs @@ -8,6 +8,8 @@ defmodule Pleroma.ScheduledActivityTest do alias Pleroma.ScheduledActivity import Pleroma.Factory + clear_config([ScheduledActivity, :enabled]) + setup context do DataCase.ensure_local_uploader(context) end @@ -61,4 +63,30 @@ test "when scheduled_at is earlier than 5 minute from now" do assert changeset.errors == [scheduled_at: {"must be at least 5 minutes from now", []}] end end + + test "creates a status from the scheduled activity" do + Pleroma.Config.put([ScheduledActivity, :enabled], true) + user = insert(:user) + + naive_datetime = + NaiveDateTime.add( + NaiveDateTime.utc_now(), + -:timer.minutes(2), + :millisecond + ) + + scheduled_activity = + insert( + :scheduled_activity, + scheduled_at: naive_datetime, + user: user, + params: %{status: "hi"} + ) + + Pleroma.Workers.Cron.ScheduledActivityWorker.perform(:opts, :pid) + + refute Repo.get(ScheduledActivity, scheduled_activity.id) + activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) + assert Pleroma.Object.normalize(activity).data["content"] == "hi" + end end From c5766a8100de465669178a98f8267425ecfe26e3 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 27 Nov 2019 13:35:02 +0300 Subject: [PATCH 04/19] moved ActivityExpiration to Oban Periodic jobs --- config/config.exs | 3 +- lib/pleroma/application.ex | 1 - .../daemons/activity_expiration_daemon.ex | 66 ------------------- .../workers/activity_expiration_worker.ex | 18 ----- .../cron/purge_expired_activities_worker.ex | 39 +++++++++++ test/activity_expiration_test.exs | 25 +++++++ .../activity_expiration_daemon_test.exs | 17 ----- 7 files changed, 66 insertions(+), 103 deletions(-) delete mode 100644 lib/pleroma/daemons/activity_expiration_daemon.ex delete mode 100644 lib/pleroma/workers/activity_expiration_worker.ex create mode 100644 lib/pleroma/workers/cron/purge_expired_activities_worker.ex delete mode 100644 test/daemons/activity_expiration_daemon_test.exs diff --git a/config/config.exs b/config/config.exs index f4f7fcce7..be8e3ee15 100644 --- a/config/config.exs +++ b/config/config.exs @@ -509,7 +509,8 @@ crontab: [ {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, {"0 * * * *", Pleroma.Workers.Cron.StatsWorker}, - {"* * * * *", Pleroma.Workers.Cron.ScheduledActivityWorker} + {"* * * * *", Pleroma.Workers.Cron.ScheduledActivityWorker}, + {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker} ] config :pleroma, :workers, diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 71670d9fc..15face96e 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -35,7 +35,6 @@ def start(_type, _args) do Pleroma.Config.TransferTask, Pleroma.Emoji, Pleroma.Captcha, - Pleroma.Daemons.ActivityExpirationDaemon, Pleroma.Plugs.RateLimiter.Supervisor ] ++ cachex_children() ++ diff --git a/lib/pleroma/daemons/activity_expiration_daemon.ex b/lib/pleroma/daemons/activity_expiration_daemon.ex deleted file mode 100644 index cab7628c4..000000000 --- a/lib/pleroma/daemons/activity_expiration_daemon.ex +++ /dev/null @@ -1,66 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Daemons.ActivityExpirationDaemon do - alias Pleroma.Activity - alias Pleroma.ActivityExpiration - alias Pleroma.Config - alias Pleroma.Repo - alias Pleroma.User - alias Pleroma.Web.CommonAPI - - require Logger - use GenServer - import Ecto.Query - - @schedule_interval :timer.minutes(1) - - def start_link(_) do - GenServer.start_link(__MODULE__, nil) - end - - @impl true - def init(_) do - if Config.get([ActivityExpiration, :enabled]) do - schedule_next() - {:ok, nil} - else - :ignore - end - end - - def perform(:execute, expiration_id) do - try do - expiration = - ActivityExpiration - |> where([e], e.id == ^expiration_id) - |> Repo.one!() - - activity = Activity.get_by_id_with_object(expiration.activity_id) - user = User.get_by_ap_id(activity.object.data["actor"]) - CommonAPI.delete(activity.id, user) - rescue - error -> - Logger.error("#{__MODULE__} Couldn't delete expired activity: #{inspect(error)}") - end - end - - @impl true - def handle_info(:perform, state) do - ActivityExpiration.due_expirations(@schedule_interval) - |> Enum.each(fn expiration -> - Pleroma.Workers.ActivityExpirationWorker.enqueue( - "activity_expiration", - %{"activity_expiration_id" => expiration.id} - ) - end) - - schedule_next() - {:noreply, state} - end - - defp schedule_next do - Process.send_after(self(), :perform, @schedule_interval) - end -end diff --git a/lib/pleroma/workers/activity_expiration_worker.ex b/lib/pleroma/workers/activity_expiration_worker.ex deleted file mode 100644 index 4e3e4195f..000000000 --- a/lib/pleroma/workers/activity_expiration_worker.ex +++ /dev/null @@ -1,18 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.ActivityExpirationWorker do - use Pleroma.Workers.WorkerHelper, queue: "activity_expiration" - - @impl Oban.Worker - def perform( - %{ - "op" => "activity_expiration", - "activity_expiration_id" => activity_expiration_id - }, - _job - ) do - Pleroma.Daemons.ActivityExpirationDaemon.perform(:execute, activity_expiration_id) - end -end diff --git a/lib/pleroma/workers/cron/purge_expired_activities_worker.ex b/lib/pleroma/workers/cron/purge_expired_activities_worker.ex new file mode 100644 index 000000000..2a7274207 --- /dev/null +++ b/lib/pleroma/workers/cron/purge_expired_activities_worker.ex @@ -0,0 +1,39 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker do + @moduledoc """ + The worker to purge expired activities. + """ + + use Oban.Worker, queue: "activity_expiration" + + alias Pleroma.Activity + alias Pleroma.ActivityExpiration + alias Pleroma.Config + alias Pleroma.User + alias Pleroma.Web.CommonAPI + + require Logger + + @interval :timer.minutes(1) + + @impl Oban.Worker + def perform(_opts, _job) do + if Config.get([ActivityExpiration, :enabled]) do + Enum.each(ActivityExpiration.due_expirations(@interval), &delete_activity/1) + end + end + + def delete_activity(expiration) do + try do + activity = Activity.get_by_id_with_object(expiration.activity_id) + user = User.get_by_ap_id(activity.object.data["actor"]) + CommonAPI.delete(activity.id, user) + rescue + error -> + Logger.error("#{__MODULE__} Couldn't delete expired activity: #{inspect(error)}") + end + end +end diff --git a/test/activity_expiration_test.exs b/test/activity_expiration_test.exs index 4948fae16..2fc593b8c 100644 --- a/test/activity_expiration_test.exs +++ b/test/activity_expiration_test.exs @@ -7,6 +7,8 @@ defmodule Pleroma.ActivityExpirationTest do alias Pleroma.ActivityExpiration import Pleroma.Factory + clear_config([ActivityExpiration, :enabled]) + test "finds activities due to be deleted only" do activity = insert(:note_activity) expiration_due = insert(:expiration_in_the_past, %{activity_id: activity.id}) @@ -24,4 +26,27 @@ test "denies expirations that don't live long enough" do now = NaiveDateTime.utc_now() assert {:error, _} = ActivityExpiration.create(activity, now) end + + test "deletes an expiration activity" do + Pleroma.Config.put([ActivityExpiration, :enabled], true) + activity = insert(:note_activity) + + naive_datetime = + NaiveDateTime.add( + NaiveDateTime.utc_now(), + -:timer.minutes(2), + :millisecond + ) + + expiration = + insert( + :expiration_in_the_past, + %{activity_id: activity.id, scheduled_at: naive_datetime} + ) + + Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker.perform(:ops, :pid) + + refute Pleroma.Repo.get(Pleroma.Activity, activity.id) + refute Pleroma.Repo.get(Pleroma.ActivityExpiration, expiration.id) + end end diff --git a/test/daemons/activity_expiration_daemon_test.exs b/test/daemons/activity_expiration_daemon_test.exs deleted file mode 100644 index b51132fb0..000000000 --- a/test/daemons/activity_expiration_daemon_test.exs +++ /dev/null @@ -1,17 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.ActivityExpirationWorkerTest do - use Pleroma.DataCase - alias Pleroma.Activity - import Pleroma.Factory - - test "deletes an activity" do - activity = insert(:note_activity) - expiration = insert(:expiration_in_the_past, %{activity_id: activity.id}) - Pleroma.Daemons.ActivityExpirationDaemon.perform(:execute, expiration.id) - - refute Repo.get(Activity, activity.id) - end -end From a4f834a687d82e7883c7dabf55b86a7e2c1dad33 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 27 Nov 2019 15:59:13 +0300 Subject: [PATCH 05/19] moved DigestEmail to Oban Periodic jobs --- config/config.exs | 18 +----- config/test.exs | 2 - lib/pleroma/application.ex | 1 - lib/pleroma/daemons/digest_email_daemon.ex | 42 -------------- lib/pleroma/scheduler.ex | 7 --- .../workers/cron/digest_emails_worker.ex | 58 +++++++++++++++++++ lib/pleroma/workers/digest_emails_worker.ex | 16 ----- mix.exs | 1 - .../cron/digest_emails_worker_test.exs} | 15 +++-- 9 files changed, 71 insertions(+), 89 deletions(-) delete mode 100644 lib/pleroma/daemons/digest_email_daemon.ex delete mode 100644 lib/pleroma/scheduler.ex create mode 100644 lib/pleroma/workers/cron/digest_emails_worker.ex delete mode 100644 lib/pleroma/workers/digest_emails_worker.ex rename test/{daemons/digest_email_daemon_test.exs => workers/cron/digest_emails_worker_test.exs} (74%) diff --git a/config/config.exs b/config/config.exs index be8e3ee15..ef9b138d1 100644 --- a/config/config.exs +++ b/config/config.exs @@ -51,20 +51,6 @@ telemetry_event: [Pleroma.Repo.Instrumenter], migration_lock: nil -scheduled_jobs = - with digest_config <- Application.get_env(:pleroma, :email_notifications)[:digest], - true <- digest_config[:active] do - [{digest_config[:schedule], {Pleroma.Daemons.DigestEmailDaemon, :perform, []}}] - else - _ -> [] - end - -config :pleroma, Pleroma.Scheduler, - global: true, - overlap: true, - timezone: :utc, - jobs: scheduled_jobs - config :pleroma, Pleroma.Captcha, enabled: false, seconds_valid: 60, @@ -510,7 +496,8 @@ {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, {"0 * * * *", Pleroma.Workers.Cron.StatsWorker}, {"* * * * *", Pleroma.Workers.Cron.ScheduledActivityWorker}, - {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker} + {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker}, + {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker} ] config :pleroma, :workers, @@ -592,7 +579,6 @@ config :pleroma, :email_notifications, digest: %{ active: false, - schedule: "0 0 * * 0", interval: 7, inactivity_threshold: 7 } diff --git a/config/test.exs b/config/test.exs index 9b737d4d7..fff709d65 100644 --- a/config/test.exs +++ b/config/test.exs @@ -68,8 +68,6 @@ queues: false, prune: :disabled -config :pleroma, Pleroma.Scheduler, jobs: [] - config :pleroma, Pleroma.ScheduledActivity, daily_user_limit: 2, total_user_limit: 3, diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 15face96e..695cc28d6 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -31,7 +31,6 @@ def start(_type, _args) do children = [ Pleroma.Repo, - Pleroma.Scheduler, Pleroma.Config.TransferTask, Pleroma.Emoji, Pleroma.Captcha, diff --git a/lib/pleroma/daemons/digest_email_daemon.ex b/lib/pleroma/daemons/digest_email_daemon.ex deleted file mode 100644 index b4c8eaad9..000000000 --- a/lib/pleroma/daemons/digest_email_daemon.ex +++ /dev/null @@ -1,42 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Daemons.DigestEmailDaemon do - alias Pleroma.Repo - alias Pleroma.Workers.DigestEmailsWorker - - import Ecto.Query - - def perform do - config = Pleroma.Config.get([:email_notifications, :digest]) - negative_interval = -Map.fetch!(config, :interval) - inactivity_threshold = Map.fetch!(config, :inactivity_threshold) - inactive_users_query = Pleroma.User.list_inactive_users_query(inactivity_threshold) - - now = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) - - from(u in inactive_users_query, - where: fragment(~s(? ->'digest' @> 'true'), u.email_notifications), - where: u.last_digest_emailed_at < datetime_add(^now, ^negative_interval, "day"), - select: u - ) - |> Repo.all() - |> Enum.each(fn user -> - DigestEmailsWorker.enqueue("digest_email", %{"user_id" => user.id}) - end) - end - - @doc """ - Send digest email to the given user. - Updates `last_digest_emailed_at` field for the user and returns the updated user. - """ - @spec perform(Pleroma.User.t()) :: Pleroma.User.t() - def perform(user) do - with %Swoosh.Email{} = email <- Pleroma.Emails.UserEmail.digest_email(user) do - Pleroma.Emails.Mailer.deliver_async(email) - end - - Pleroma.User.touch_last_digest_emailed_at(user) - end -end diff --git a/lib/pleroma/scheduler.ex b/lib/pleroma/scheduler.ex deleted file mode 100644 index d84cd99ad..000000000 --- a/lib/pleroma/scheduler.ex +++ /dev/null @@ -1,7 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Scheduler do - use Quantum.Scheduler, otp_app: :pleroma -end diff --git a/lib/pleroma/workers/cron/digest_emails_worker.ex b/lib/pleroma/workers/cron/digest_emails_worker.ex new file mode 100644 index 000000000..0a00129df --- /dev/null +++ b/lib/pleroma/workers/cron/digest_emails_worker.ex @@ -0,0 +1,58 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.DigestEmailsWorker do + @moduledoc """ + The worker to send digest emails. + """ + + use Oban.Worker, queue: "digest_emails" + + alias Pleroma.Config + alias Pleroma.Emails + alias Pleroma.Repo + alias Pleroma.User + + import Ecto.Query + + require Logger + + @impl Oban.Worker + def perform(_opts, _job) do + config = Config.get([:email_notifications, :digest]) + + if config[:active] do + negative_interval = -Map.fetch!(config, :interval) + inactivity_threshold = Map.fetch!(config, :inactivity_threshold) + inactive_users_query = User.list_inactive_users_query(inactivity_threshold) + + now = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) + + from(u in inactive_users_query, + where: fragment(~s(? ->'digest' @> 'true'), u.email_notifications), + where: u.last_digest_emailed_at < datetime_add(^now, ^negative_interval, "day"), + select: u + ) + |> Repo.all() + |> send_emails + end + end + + def send_emails(users) do + Enum.each(users, &send_email/1) + end + + @doc """ + Send digest email to the given user. + Updates `last_digest_emailed_at` field for the user and returns the updated user. + """ + @spec send_email(User.t()) :: User.t() + def send_email(user) do + with %Swoosh.Email{} = email <- Emails.UserEmail.digest_email(user) do + Emails.Mailer.deliver_async(email) + end + + User.touch_last_digest_emailed_at(user) + end +end diff --git a/lib/pleroma/workers/digest_emails_worker.ex b/lib/pleroma/workers/digest_emails_worker.ex deleted file mode 100644 index 3e5a836d0..000000000 --- a/lib/pleroma/workers/digest_emails_worker.ex +++ /dev/null @@ -1,16 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.DigestEmailsWorker do - alias Pleroma.User - - use Pleroma.Workers.WorkerHelper, queue: "digest_emails" - - @impl Oban.Worker - def perform(%{"op" => "digest_email", "user_id" => user_id}, _job) do - user_id - |> User.get_cached_by_id() - |> Pleroma.Daemons.DigestEmailDaemon.perform() - end -end diff --git a/mix.exs b/mix.exs index 31deae96e..23d981dac 100644 --- a/mix.exs +++ b/mix.exs @@ -101,7 +101,6 @@ defp deps do {:phoenix_pubsub, "~> 1.1"}, {:phoenix_ecto, "~> 4.0"}, {:ecto_sql, "~> 3.2"}, - {:postgrex, ">= 0.13.5"}, {:oban, "~> 0.11.1"}, {:quantum, "~> 2.3"}, {:gettext, "~> 0.15"}, diff --git a/test/daemons/digest_email_daemon_test.exs b/test/workers/cron/digest_emails_worker_test.exs similarity index 74% rename from test/daemons/digest_email_daemon_test.exs rename to test/workers/cron/digest_emails_worker_test.exs index faf592d5f..073615d9e 100644 --- a/test/daemons/digest_email_daemon_test.exs +++ b/test/workers/cron/digest_emails_worker_test.exs @@ -2,16 +2,24 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.DigestEmailDaemonTest do +defmodule Pleroma.Workers.Cron.DigestEmailsWorkerTest do use Pleroma.DataCase + import Pleroma.Factory - alias Pleroma.Daemons.DigestEmailDaemon alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.CommonAPI + clear_config([:email_notifications, :digest]) + test "it sends digest emails" do + Pleroma.Config.put([:email_notifications, :digest], %{ + active: true, + inactivity_threshold: 7, + interval: 7 + }) + user = insert(:user) date = @@ -23,8 +31,7 @@ test "it sends digest emails" do {:ok, _} = User.switch_email_notifications(user2, "digest", true) CommonAPI.post(user, %{"status" => "hey @#{user2.nickname}!"}) - DigestEmailDaemon.perform() - ObanHelpers.perform_all() + Pleroma.Workers.Cron.DigestEmailsWorker.perform(:opts, :pid) # Performing job(s) enqueued at previous step ObanHelpers.perform_all() From d55859f5a4cabbe7849dc3f39079b17bd450a303 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 27 Nov 2019 16:35:12 +0300 Subject: [PATCH 06/19] remove `quantum` package --- config/config.exs | 1 - mix.exs | 3 +-- mix.lock | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/config/config.exs b/config/config.exs index ef9b138d1..54d167a82 100644 --- a/config/config.exs +++ b/config/config.exs @@ -607,7 +607,6 @@ activity_pub: nil, activity_pub_question: 30_000 -config :swarm, node_blacklist: [~r/myhtml_.*$/] # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/mix.exs b/mix.exs index 23d981dac..72e37cfe6 100644 --- a/mix.exs +++ b/mix.exs @@ -63,7 +63,7 @@ def copy_nginx_config(%{path: target_path} = release) do def application do [ mod: {Pleroma.Application, []}, - extra_applications: [:logger, :runtime_tools, :comeonin, :quack, :fast_sanitize, :swarm], + extra_applications: [:logger, :runtime_tools, :comeonin, :quack, :fast_sanitize], included_applications: [:ex_syslogger] ] end @@ -102,7 +102,6 @@ defp deps do {:phoenix_ecto, "~> 4.0"}, {:ecto_sql, "~> 3.2"}, {:oban, "~> 0.11.1"}, - {:quantum, "~> 2.3"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, {:pbkdf2_elixir, "~> 0.12.3"}, diff --git a/mix.lock b/mix.lock index c212b39e0..ff64b1d75 100644 --- a/mix.lock +++ b/mix.lock @@ -88,12 +88,10 @@ "prometheus_phoenix": {:hex, :prometheus_phoenix, "1.3.0", "c4b527e0b3a9ef1af26bdcfbfad3998f37795b9185d475ca610fe4388fdd3bb5", [:mix], [{:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm"}, "quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm"}, - "quantum": {:hex, :quantum, "2.3.4", "72a0e8855e2adc101459eac8454787cb74ab4169de6ca50f670e72142d4960e9", [:mix], [{:calendar, "~> 0.17", [hex: :calendar, repo: "hexpm", optional: true]}, {:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.12", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:swarm, "~> 3.3", [hex: :swarm, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: true]}], "hexpm"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]}, "remote_ip": {:git, "https://git.pleroma.social/pleroma/remote_ip.git", "825dc00aaba5a1b7c4202a532b696b595dd3bcb3", [ref: "825dc00aaba5a1b7c4202a532b696b595dd3bcb3"]}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.5", "6eaf7ad16cb568bb01753dbbd7a95ff8b91c7979482b95f38443fe2c8852a79b", [:make, :mix, :rebar3], [], "hexpm"}, - "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm"}, "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.23.5", "bfd9404bbf5069b1be2ffd317923ce57e58b332e25dbca2a35dedd7820dfee5a", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, From 1403a1441de36e8a58da5e996c208a9c10f65440 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Thu, 28 Nov 2019 09:02:23 +0300 Subject: [PATCH 07/19] added tests --- .../cron/clear_oauth_token_worker_test.exs | 22 +++++++++++ .../purge_expired_activities_worker_test.exs | 34 +++++++++++++++++ .../cron/scheduled_activity_worker_test.exs | 37 +++++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 test/workers/cron/clear_oauth_token_worker_test.exs create mode 100644 test/workers/cron/purge_expired_activities_worker_test.exs create mode 100644 test/workers/cron/scheduled_activity_worker_test.exs diff --git a/test/workers/cron/clear_oauth_token_worker_test.exs b/test/workers/cron/clear_oauth_token_worker_test.exs new file mode 100644 index 000000000..adea47326 --- /dev/null +++ b/test/workers/cron/clear_oauth_token_worker_test.exs @@ -0,0 +1,22 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.ClearOauthTokenWorkerTest do + use Pleroma.DataCase + + import Pleroma.Factory + alias Pleroma.Workers.Cron.ClearOauthTokenWorker + + clear_config([:oauth2, :clean_expired_tokens]) + + test "deletes expired tokens" do + insert(:oauth_token, + valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), -60 * 10) + ) + + Pleroma.Config.put([:oauth2, :clean_expired_tokens], true) + ClearOauthTokenWorker.perform(:opts, :job) + assert Pleroma.Repo.all(Pleroma.Web.OAuth.Token) == [] + end +end diff --git a/test/workers/cron/purge_expired_activities_worker_test.exs b/test/workers/cron/purge_expired_activities_worker_test.exs new file mode 100644 index 000000000..07980bcd0 --- /dev/null +++ b/test/workers/cron/purge_expired_activities_worker_test.exs @@ -0,0 +1,34 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.PurgeExpiredActivitiesWorkerTest do + use Pleroma.DataCase + alias Pleroma.ActivityExpiration + import Pleroma.Factory + + clear_config([ActivityExpiration, :enabled]) + + test "deletes an expiration activity" do + Pleroma.Config.put([ActivityExpiration, :enabled], true) + activity = insert(:note_activity) + + naive_datetime = + NaiveDateTime.add( + NaiveDateTime.utc_now(), + -:timer.minutes(2), + :millisecond + ) + + expiration = + insert( + :expiration_in_the_past, + %{activity_id: activity.id, scheduled_at: naive_datetime} + ) + + Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker.perform(:ops, :pid) + + refute Pleroma.Repo.get(Pleroma.Activity, activity.id) + refute Pleroma.Repo.get(Pleroma.ActivityExpiration, expiration.id) + end +end diff --git a/test/workers/cron/scheduled_activity_worker_test.exs b/test/workers/cron/scheduled_activity_worker_test.exs new file mode 100644 index 000000000..6f17d6f6c --- /dev/null +++ b/test/workers/cron/scheduled_activity_worker_test.exs @@ -0,0 +1,37 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.ScheduledActivityWorkerTest do + use Pleroma.DataCase + alias Pleroma.ScheduledActivity + import Pleroma.Factory + + clear_config([ScheduledActivity, :enabled]) + + test "creates a status from the scheduled activity" do + Pleroma.Config.put([ScheduledActivity, :enabled], true) + user = insert(:user) + + naive_datetime = + NaiveDateTime.add( + NaiveDateTime.utc_now(), + -:timer.minutes(2), + :millisecond + ) + + scheduled_activity = + insert( + :scheduled_activity, + scheduled_at: naive_datetime, + user: user, + params: %{status: "hi"} + ) + + Pleroma.Workers.Cron.ScheduledActivityWorker.perform(:opts, :pid) + + refute Repo.get(ScheduledActivity, scheduled_activity.id) + activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) + assert Pleroma.Object.normalize(activity).data["content"] == "hi" + end +end From 652cc6ba4b7a0494cc96ef4a9bfcaa3b5e5be60e Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 3 Dec 2019 21:30:10 +0300 Subject: [PATCH 08/19] updated ScheduledActivity --- config/config.exs | 1 - lib/pleroma/scheduled_activity.ex | 66 +++++++++++++++---- .../scheduled_activity_controller.ex | 3 +- .../controllers/status_controller.ex | 19 +++--- .../{cron => }/scheduled_activity_worker.ex | 23 ++++--- test/scheduled_activity_test.exs | 6 +- test/support/helpers.ex | 6 ++ .../scheduled_activity_controller_test.exs | 53 ++++++++++++++- .../cron/scheduled_activity_worker_test.exs | 37 ----------- 9 files changed, 142 insertions(+), 72 deletions(-) rename lib/pleroma/workers/{cron => }/scheduled_activity_worker.ex (59%) delete mode 100644 test/workers/cron/scheduled_activity_worker_test.exs diff --git a/config/config.exs b/config/config.exs index e7e17669e..259529f97 100644 --- a/config/config.exs +++ b/config/config.exs @@ -496,7 +496,6 @@ crontab: [ {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, {"0 * * * *", Pleroma.Workers.Cron.StatsWorker}, - {"* * * * *", Pleroma.Workers.Cron.ScheduledActivityWorker}, {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker}, {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker} ] diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index fea2cf3ff..96fa6a987 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -5,11 +5,13 @@ defmodule Pleroma.ScheduledActivity do use Ecto.Schema + alias Ecto.Multi alias Pleroma.Config alias Pleroma.Repo alias Pleroma.ScheduledActivity alias Pleroma.User alias Pleroma.Web.CommonAPI.Utils + alias Pleroma.Workers.ScheduledActivityWorker import Ecto.Query import Ecto.Changeset @@ -105,14 +107,29 @@ def far_enough?(scheduled_at) do end def new(%User{} = user, attrs) do - %ScheduledActivity{user_id: user.id} - |> changeset(attrs) + changeset(%ScheduledActivity{user_id: user.id}, attrs) end + @doc """ + Creates ScheduledActivity and add to queue to perform at scheduled_at date + """ + @spec create(User.t(), map()) :: {:ok, ScheduledActivity.t()} | {:error, Ecto.Changeset.t()} def create(%User{} = user, attrs) do - user - |> new(attrs) - |> Repo.insert() + Multi.new() + |> Multi.insert(:scheduled_activity, new(user, attrs)) + |> Multi.run(:scheduled_activity_job, fn _repo, %{scheduled_activity: activity} -> + %{activity_id: activity.id} + |> ScheduledActivityWorker.new(scheduled_at: activity.scheduled_at) + |> Oban.insert() + end) + |> Repo.transaction() + |> case do + {:ok, %{scheduled_activity: scheduled_activity}} -> + {:ok, scheduled_activity} + + {:error, _, changeset, _} -> + {:error, changeset} + end end def get(%User{} = user, scheduled_activity_id) do @@ -122,15 +139,35 @@ def get(%User{} = user, scheduled_activity_id) do |> Repo.one() end - def update(%ScheduledActivity{} = scheduled_activity, attrs) do - scheduled_activity - |> update_changeset(attrs) - |> Repo.update() + @spec update(ScheduledActivity.t(), map()) :: + {:ok, ScheduledActivity.t()} | {:error, Ecto.Changeset.t()} + def update(%ScheduledActivity{id: id} = scheduled_activity, attrs) do + with {:error, %Ecto.Changeset{valid?: true} = changeset} <- + {:error, update_changeset(scheduled_activity, attrs)} do + Multi.new() + |> Multi.update(:scheduled_activity, changeset) + |> Multi.update_all(:scheduled_job, job_query(id), + set: [scheduled_at: changeset.changes[:scheduled_at]] + ) + |> Repo.transaction() + |> case do + {:ok, %{scheduled_activity: scheduled_activity}} -> + {:ok, scheduled_activity} + + {:error, _, changeset, _} -> + {:error, changeset} + end + end + end + + def delete_job(%ScheduledActivity{id: id} = _scheduled_activity) do + id + |> job_query + |> Repo.delete_all() end def delete(%ScheduledActivity{} = scheduled_activity) do - scheduled_activity - |> Repo.delete() + Repo.delete(scheduled_activity) end def delete(id) when is_binary(id) or is_integer(id) do @@ -158,4 +195,11 @@ def due_activities(offset \\ 0) do |> where([sa], sa.scheduled_at < ^naive_datetime) |> Repo.all() end + + def job_query(scheduled_activity_id) do + from(j in Oban.Job, + where: j.queue == "scheduled_activities", + where: fragment("args ->> 'activity_id' = ?::text", ^to_string(scheduled_activity_id)) + ) + end end diff --git a/lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex b/lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex index ff9276541..4f9a8bdbe 100644 --- a/lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex @@ -45,7 +45,8 @@ def update(%{assigns: %{scheduled_activity: scheduled_activity}} = conn, params) @doc "DELETE /api/v1/scheduled_statuses/:id" def delete(%{assigns: %{scheduled_activity: scheduled_activity}} = conn, _params) do - with {:ok, scheduled_activity} <- ScheduledActivity.delete(scheduled_activity) do + with {:ok, scheduled_activity} <- ScheduledActivity.delete(scheduled_activity), + _ <- ScheduledActivity.delete_job(scheduled_activity) do render(conn, "show.json", scheduled_activity: scheduled_activity) end end diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 74b223cf4..d70749dfa 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -124,15 +124,18 @@ def create( ) do params = Map.put(params, "in_reply_to_status_id", params["in_reply_to_id"]) - if ScheduledActivity.far_enough?(scheduled_at) do - with {:ok, scheduled_activity} <- - ScheduledActivity.create(user, %{"params" => params, "scheduled_at" => scheduled_at}) do - conn - |> put_view(ScheduledActivityView) - |> render("show.json", scheduled_activity: scheduled_activity) - end + with {:far_enough, true} <- {:far_enough, ScheduledActivity.far_enough?(scheduled_at)}, + attrs <- %{"params" => params, "scheduled_at" => scheduled_at}, + {:ok, scheduled_activity} <- ScheduledActivity.create(user, attrs) do + conn + |> put_view(ScheduledActivityView) + |> render("show.json", scheduled_activity: scheduled_activity) else - create(conn, Map.drop(params, ["scheduled_at"])) + {:far_enough, _} -> + create(conn, Map.drop(params, ["scheduled_at"])) + + error -> + error end end diff --git a/lib/pleroma/workers/cron/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex similarity index 59% rename from lib/pleroma/workers/cron/scheduled_activity_worker.ex rename to lib/pleroma/workers/scheduled_activity_worker.ex index 407ab687a..5109d7f75 100644 --- a/lib/pleroma/workers/cron/scheduled_activity_worker.ex +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -2,12 +2,13 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Workers.Cron.ScheduledActivityWorker do +defmodule Pleroma.Workers.ScheduledActivityWorker do @moduledoc """ - The worker to post scheduled actvities. + The worker to post scheduled activity. """ - use Oban.Worker, queue: "scheduled_activities" + use Pleroma.Workers.WorkerHelper, queue: "scheduled_activities" + alias Pleroma.Config alias Pleroma.ScheduledActivity alias Pleroma.User @@ -15,18 +16,20 @@ defmodule Pleroma.Workers.Cron.ScheduledActivityWorker do require Logger - @schedule_interval :timer.minutes(1) - @impl Oban.Worker - def perform(_opts, _job) do + def perform(%{"activity_id" => activity_id}, _job) do if Config.get([ScheduledActivity, :enabled]) do - @schedule_interval - |> ScheduledActivity.due_activities() - |> Enum.each(&post_activity/1) + case Pleroma.Repo.get(ScheduledActivity, activity_id) do + %ScheduledActivity{} = scheduled_activity -> + post_activity(scheduled_activity) + + _ -> + Logger.error("#{__MODULE__} Couldn't find scheduled activity: #{activity_id}") + end end end - def post_activity(scheduled_activity) do + defp post_activity(%ScheduledActivity{} = scheduled_activity) do try do {:ok, scheduled_activity} = ScheduledActivity.delete(scheduled_activity) %User{} = user = User.get_cached_by_id(scheduled_activity.user_id) diff --git a/test/scheduled_activity_test.exs b/test/scheduled_activity_test.exs index d3d05745f..663cfdd34 100644 --- a/test/scheduled_activity_test.exs +++ b/test/scheduled_activity_test.exs @@ -26,6 +26,7 @@ test "when daily user limit is exceeded" do attrs = %{params: %{}, scheduled_at: today} {:ok, _} = ScheduledActivity.create(user, attrs) {:ok, _} = ScheduledActivity.create(user, attrs) + {:error, changeset} = ScheduledActivity.create(user, attrs) assert changeset.errors == [scheduled_at: {"daily limit exceeded", []}] end @@ -83,7 +84,10 @@ test "creates a status from the scheduled activity" do params: %{status: "hi"} ) - Pleroma.Workers.Cron.ScheduledActivityWorker.perform(:opts, :pid) + Pleroma.Workers.ScheduledActivityWorker.perform( + %{"activity_id" => scheduled_activity.id}, + :pid + ) refute Repo.get(ScheduledActivity, scheduled_activity.id) activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) diff --git a/test/support/helpers.ex b/test/support/helpers.ex index ce39dd9d8..ec556a916 100644 --- a/test/support/helpers.ex +++ b/test/support/helpers.ex @@ -53,6 +53,12 @@ defmacro __using__(_opts) do clear_config_all: 2 ] + def to_datetime(naive_datetime) do + naive_datetime + |> DateTime.from_naive!("Etc/UTC") + |> DateTime.truncate(:second) + end + def collect_ids(collection) do collection |> Enum.map(& &1.id) diff --git a/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs b/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs index ae5fee2bc..5f3a376be 100644 --- a/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs +++ b/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs @@ -9,6 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do alias Pleroma.ScheduledActivity import Pleroma.Factory + import Ecto.Query test "shows scheduled activities", %{conn: conn} do user = insert(:user) @@ -68,7 +69,30 @@ test "shows a scheduled activity", %{conn: conn} do test "updates a scheduled activity", %{conn: conn} do user = insert(:user) - scheduled_activity = insert(:scheduled_activity, user: user) + + scheduled_at = + NaiveDateTime.add( + NaiveDateTime.utc_now(), + :timer.minutes(60), + :millisecond + ) + + {:ok, scheduled_activity} = + ScheduledActivity.create( + user, + %{ + scheduled_at: scheduled_at, + params: build(:note).data + } + ) + + scheduled_activity_job = + Repo.one(from(j in Oban.Job, where: j.queue == "scheduled_activities")) + + assert scheduled_activity_job.args == %{"activity_id" => scheduled_activity.id} + + assert DateTime.truncate(scheduled_activity_job.scheduled_at, :second) == + to_datetime(scheduled_at) new_scheduled_at = NaiveDateTime.add(NaiveDateTime.utc_now(), :timer.minutes(120), :millisecond) @@ -82,6 +106,10 @@ test "updates a scheduled activity", %{conn: conn} do assert %{"scheduled_at" => expected_scheduled_at} = json_response(res_conn, 200) assert expected_scheduled_at == Pleroma.Web.CommonAPI.Utils.to_masto_date(new_scheduled_at) + scheduled_activity_job = refresh_record(scheduled_activity_job) + + assert DateTime.truncate(scheduled_activity_job.scheduled_at, :second) == + to_datetime(new_scheduled_at) res_conn = conn @@ -93,7 +121,25 @@ test "updates a scheduled activity", %{conn: conn} do test "deletes a scheduled activity", %{conn: conn} do user = insert(:user) - scheduled_activity = insert(:scheduled_activity, user: user) + + {:ok, scheduled_activity} = + ScheduledActivity.create( + user, + %{ + scheduled_at: + NaiveDateTime.add( + NaiveDateTime.utc_now(), + :timer.minutes(60), + :millisecond + ), + params: build(:note).data + } + ) + + scheduled_activity_job = + Repo.one(from(j in Oban.Job, where: j.queue == "scheduled_activities")) + + assert scheduled_activity_job.args == %{"activity_id" => scheduled_activity.id} res_conn = conn @@ -101,7 +147,8 @@ test "deletes a scheduled activity", %{conn: conn} do |> delete("/api/v1/scheduled_statuses/#{scheduled_activity.id}") assert %{} = json_response(res_conn, 200) - assert nil == Repo.get(ScheduledActivity, scheduled_activity.id) + refute Repo.get(ScheduledActivity, scheduled_activity.id) + refute Repo.get(Oban.Job, scheduled_activity_job.id) res_conn = conn diff --git a/test/workers/cron/scheduled_activity_worker_test.exs b/test/workers/cron/scheduled_activity_worker_test.exs deleted file mode 100644 index 6f17d6f6c..000000000 --- a/test/workers/cron/scheduled_activity_worker_test.exs +++ /dev/null @@ -1,37 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.Cron.ScheduledActivityWorkerTest do - use Pleroma.DataCase - alias Pleroma.ScheduledActivity - import Pleroma.Factory - - clear_config([ScheduledActivity, :enabled]) - - test "creates a status from the scheduled activity" do - Pleroma.Config.put([ScheduledActivity, :enabled], true) - user = insert(:user) - - naive_datetime = - NaiveDateTime.add( - NaiveDateTime.utc_now(), - -:timer.minutes(2), - :millisecond - ) - - scheduled_activity = - insert( - :scheduled_activity, - scheduled_at: naive_datetime, - user: user, - params: %{status: "hi"} - ) - - Pleroma.Workers.Cron.ScheduledActivityWorker.perform(:opts, :pid) - - refute Repo.get(ScheduledActivity, scheduled_activity.id) - activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) - assert Pleroma.Object.normalize(activity).data["content"] == "hi" - end -end From 469c0956cd937b28a38f8c982b670207ec14b843 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 4 Dec 2019 09:43:01 +0300 Subject: [PATCH 09/19] update excoveralls --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 618864eff..fc84ae962 100644 --- a/mix.exs +++ b/mix.exs @@ -156,7 +156,7 @@ defp deps do {:esshd, "~> 0.1.0", runtime: Application.get_env(:esshd, :enabled, false)}, {:ex_const, "~> 0.2"}, {:plug_static_index_html, "~> 1.0.0"}, - {:excoveralls, "~> 0.11.1", only: :test}, + {:excoveralls, "~> 0.12.1", only: :test}, {:flake_id, "~> 0.1.0"}, {:remote_ip, git: "https://git.pleroma.social/pleroma/remote_ip.git", diff --git a/mix.lock b/mix.lock index 2e12d9b73..c76874a4d 100644 --- a/mix.lock +++ b/mix.lock @@ -34,7 +34,7 @@ "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, "ex_machina": {:hex, :ex_machina, "2.3.0", "92a5ad0a8b10ea6314b876a99c8c9e3f25f4dde71a2a835845b136b9adaf199a", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm"}, "ex_syslogger": {:git, "https://github.com/slashmili/ex_syslogger.git", "f3963399047af17e038897c69e20d552e6899e1d", [tag: "1.4.0"]}, - "excoveralls": {:hex, :excoveralls, "0.11.2", "0c6f2c8db7683b0caa9d490fb8125709c54580b4255ffa7ad35f3264b075a643", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, + "excoveralls": {:hex, :excoveralls, "0.12.1", "a553c59f6850d0aff3770e4729515762ba7c8e41eedde03208182a8dc9d0ce07", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, "fast_html": {:hex, :fast_html, "0.99.4", "d80812664f0429607e1d880fba0ef04da87a2e4fa596701bcaae17953535695c", [:make, :mix], [], "hexpm"}, "fast_sanitize": {:hex, :fast_sanitize, "0.1.4", "6c2e7203ca2f8275527a3021ba6e9d5d4ee213a47dc214a97c128737c9e56df1", [:mix], [{:fast_html, "~> 0.99", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, From 3a0a400fe154f44ab37b79eeb3a07f919517743e Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 4 Dec 2019 09:53:01 +0300 Subject: [PATCH 10/19] add @type to ScheduledActivity --- lib/pleroma/scheduled_activity.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index 96fa6a987..ea78dce6a 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -16,6 +16,8 @@ defmodule Pleroma.ScheduledActivity do import Ecto.Query import Ecto.Changeset + @type t :: %__MODULE__{} + @min_offset :timer.minutes(5) schema "scheduled_activities" do From c0d572d0bf842fae08c609aa58c82554bee3a263 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 4 Dec 2019 17:28:57 +0300 Subject: [PATCH 11/19] added test --- test/scheduled_activity_test.exs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/scheduled_activity_test.exs b/test/scheduled_activity_test.exs index 663cfdd34..b367ae5fb 100644 --- a/test/scheduled_activity_test.exs +++ b/test/scheduled_activity_test.exs @@ -24,9 +24,13 @@ test "when daily user limit is exceeded" do |> NaiveDateTime.to_iso8601() attrs = %{params: %{}, scheduled_at: today} - {:ok, _} = ScheduledActivity.create(user, attrs) - {:ok, _} = ScheduledActivity.create(user, attrs) + {:ok, sa1} = ScheduledActivity.create(user, attrs) + {:ok, sa2} = ScheduledActivity.create(user, attrs) + jobs = + Repo.all(from(j in Oban.Job, where: j.queue == "scheduled_activities", select: j.args)) + + assert jobs == [%{"activity_id" => sa1.id}, %{"activity_id" => sa2.id}] {:error, changeset} = ScheduledActivity.create(user, attrs) assert changeset.errors == [scheduled_at: {"daily limit exceeded", []}] end From 3c3bba0b7c65187b3270ef3402442cf870a55198 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 4 Dec 2019 21:18:05 +0300 Subject: [PATCH 12/19] fix ScheduledActivity --- lib/pleroma/scheduled_activity.ex | 17 ++++++--- test/scheduled_activity_test.exs | 36 ++++++++++++++++++- .../scheduled_activity_controller_test.exs | 4 +++ 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index ea78dce6a..d01100702 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -119,11 +119,7 @@ def new(%User{} = user, attrs) do def create(%User{} = user, attrs) do Multi.new() |> Multi.insert(:scheduled_activity, new(user, attrs)) - |> Multi.run(:scheduled_activity_job, fn _repo, %{scheduled_activity: activity} -> - %{activity_id: activity.id} - |> ScheduledActivityWorker.new(scheduled_at: activity.scheduled_at) - |> Oban.insert() - end) + |> maybe_add_jobs(Config.get([ScheduledActivity, :enabled])) |> Repo.transaction() |> case do {:ok, %{scheduled_activity: scheduled_activity}} -> @@ -134,6 +130,17 @@ def create(%User{} = user, attrs) do end end + defp maybe_add_jobs(multi, true) do + multi + |> Multi.run(:scheduled_activity_job, fn _repo, %{scheduled_activity: activity} -> + %{activity_id: activity.id} + |> ScheduledActivityWorker.new(scheduled_at: activity.scheduled_at) + |> Oban.insert() + end) + end + + defp maybe_add_jobs(multi, _), do: multi + def get(%User{} = user, scheduled_activity_id) do ScheduledActivity |> where(user_id: ^user.id) diff --git a/test/scheduled_activity_test.exs b/test/scheduled_activity_test.exs index b367ae5fb..d2c5f5aa2 100644 --- a/test/scheduled_activity_test.exs +++ b/test/scheduled_activity_test.exs @@ -15,7 +15,8 @@ defmodule Pleroma.ScheduledActivityTest do end describe "creation" do - test "when daily user limit is exceeded" do + test "scheduled activities with jobs when ScheduledActivity enabled" do + Pleroma.Config.put([ScheduledActivity, :enabled], true) user = insert(:user) today = @@ -31,6 +32,39 @@ test "when daily user limit is exceeded" do Repo.all(from(j in Oban.Job, where: j.queue == "scheduled_activities", select: j.args)) assert jobs == [%{"activity_id" => sa1.id}, %{"activity_id" => sa2.id}] + end + + test "scheduled activities without jobs when ScheduledActivity disabled" do + Pleroma.Config.put([ScheduledActivity, :enabled], false) + user = insert(:user) + + today = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(6), :millisecond) + |> NaiveDateTime.to_iso8601() + + attrs = %{params: %{}, scheduled_at: today} + {:ok, _sa1} = ScheduledActivity.create(user, attrs) + {:ok, _sa2} = ScheduledActivity.create(user, attrs) + + jobs = + Repo.all(from(j in Oban.Job, where: j.queue == "scheduled_activities", select: j.args)) + + assert jobs == [] + end + + test "when daily user limit is exceeded" do + user = insert(:user) + + today = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(6), :millisecond) + |> NaiveDateTime.to_iso8601() + + attrs = %{params: %{}, scheduled_at: today} + {:ok, _} = ScheduledActivity.create(user, attrs) + {:ok, _} = ScheduledActivity.create(user, attrs) + {:error, changeset} = ScheduledActivity.create(user, attrs) assert changeset.errors == [scheduled_at: {"daily limit exceeded", []}] end diff --git a/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs b/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs index 5f3a376be..478631a12 100644 --- a/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs +++ b/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs @@ -11,6 +11,8 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do import Pleroma.Factory import Ecto.Query + clear_config([ScheduledActivity, :enabled]) + test "shows scheduled activities", %{conn: conn} do user = insert(:user) scheduled_activity_id1 = insert(:scheduled_activity, user: user).id |> to_string() @@ -68,6 +70,7 @@ test "shows a scheduled activity", %{conn: conn} do end test "updates a scheduled activity", %{conn: conn} do + Pleroma.Config.put([ScheduledActivity, :enabled], true) user = insert(:user) scheduled_at = @@ -120,6 +123,7 @@ test "updates a scheduled activity", %{conn: conn} do end test "deletes a scheduled activity", %{conn: conn} do + Pleroma.Config.put([ScheduledActivity, :enabled], true) user = insert(:user) {:ok, scheduled_activity} = From d43c9ab286c410abfe903b56911888d75f492155 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 22 Jan 2020 16:58:43 +0000 Subject: [PATCH 13/19] Apply suggestion to docs/configuration/cheatsheet.md --- docs/configuration/cheatsheet.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index f690259bf..22f099581 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -515,7 +515,7 @@ Pleroma has the following queues: #### Oban.Cron -Pleroma has the periodic jobs: +Pleroma has these periodic job workers: `Pleroma.Workers.Cron.ClearOauthTokenWorker` - the job to clean an expired oauth tokens. From e761d76e4fe95a8fadd8eb05d5c55ef5fc4bc8ec Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 22 Jan 2020 16:58:50 +0000 Subject: [PATCH 14/19] Apply suggestion to docs/configuration/cheatsheet.md --- docs/configuration/cheatsheet.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 22f099581..48d980aea 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -517,7 +517,7 @@ Pleroma has the following queues: Pleroma has these periodic job workers: -`Pleroma.Workers.Cron.ClearOauthTokenWorker` - the job to clean an expired oauth tokens. +`Pleroma.Workers.Cron.ClearOauthTokenWorker` - a job worker to cleanup expired oauth tokens. Example: From 6f77b7742837f476fb2bf732e5c4ba6bc4fb3799 Mon Sep 17 00:00:00 2001 From: Maksim Date: Thu, 23 Jan 2020 05:49:00 +0000 Subject: [PATCH 15/19] Apply suggestion to lib/pleroma/workers/cron/clear_oauth_token_worker.ex --- lib/pleroma/workers/cron/clear_oauth_token_worker.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/workers/cron/clear_oauth_token_worker.ex b/lib/pleroma/workers/cron/clear_oauth_token_worker.ex index 1a4cdc198..a24407874 100644 --- a/lib/pleroma/workers/cron/clear_oauth_token_worker.ex +++ b/lib/pleroma/workers/cron/clear_oauth_token_worker.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Workers.Cron.ClearOauthTokenWorker do @moduledoc """ - The worker to clean an expired oauth tokens. + The worker to cleanup expired oAuth tokens. """ use Oban.Worker, queue: "background" From ce7c887a27ad5af59f540650637a50da7f91fa52 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Thu, 23 Jan 2020 11:05:08 +0300 Subject: [PATCH 16/19] removed try/rescue --- lib/pleroma/scheduled_activity.ex | 2 +- .../cron/purge_expired_activities_worker.ex | 21 +++++--- .../workers/scheduled_activity_worker.ex | 12 ++--- test/scheduled_activity_test.exs | 29 ----------- .../purge_expired_activities_worker_test.exs | 22 ++++++++ .../scheduled_activity_worker_test.exs | 52 +++++++++++++++++++ 6 files changed, 95 insertions(+), 43 deletions(-) create mode 100644 test/workers/scheduled_activity_worker_test.exs diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index d01100702..68da0550c 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -156,7 +156,7 @@ def update(%ScheduledActivity{id: id} = scheduled_activity, attrs) do Multi.new() |> Multi.update(:scheduled_activity, changeset) |> Multi.update_all(:scheduled_job, job_query(id), - set: [scheduled_at: changeset.changes[:scheduled_at]] + set: [scheduled_at: get_field(changeset, :scheduled_at)] ) |> Repo.transaction() |> case do diff --git a/lib/pleroma/workers/cron/purge_expired_activities_worker.ex b/lib/pleroma/workers/cron/purge_expired_activities_worker.ex index 2a7274207..7a52860a9 100644 --- a/lib/pleroma/workers/cron/purge_expired_activities_worker.ex +++ b/lib/pleroma/workers/cron/purge_expired_activities_worker.ex @@ -26,14 +26,21 @@ def perform(_opts, _job) do end end - def delete_activity(expiration) do - try do - activity = Activity.get_by_id_with_object(expiration.activity_id) - user = User.get_by_ap_id(activity.object.data["actor"]) + def delete_activity(%ActivityExpiration{activity_id: activity_id}) do + with {:activity, %Activity{} = activity} <- + {:activity, Activity.get_by_id_with_object(activity_id)}, + {:user, %User{} = user} <- {:user, User.get_by_ap_id(activity.object.data["actor"])} do CommonAPI.delete(activity.id, user) - rescue - error -> - Logger.error("#{__MODULE__} Couldn't delete expired activity: #{inspect(error)}") + else + {:activity, _} -> + Logger.error( + "#{__MODULE__} Couldn't delete expired activity: not found activity ##{activity_id}" + ) + + {:user, _} -> + Logger.error( + "#{__MODULE__} Couldn't delete expired activity: not found actorof ##{activity_id}" + ) end end end diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex index 5109d7f75..bd41ab4ce 100644 --- a/lib/pleroma/workers/scheduled_activity_worker.ex +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -29,12 +29,12 @@ def perform(%{"activity_id" => activity_id}, _job) do end end - defp post_activity(%ScheduledActivity{} = scheduled_activity) do - try do - {:ok, scheduled_activity} = ScheduledActivity.delete(scheduled_activity) - %User{} = user = User.get_cached_by_id(scheduled_activity.user_id) - {:ok, _result} = CommonAPI.post(user, scheduled_activity.params) - rescue + defp post_activity(%ScheduledActivity{user_id: user_id, params: params} = scheduled_activity) do + with {:delete, {:ok, _}} <- {:delete, ScheduledActivity.delete(scheduled_activity)}, + {:user, %User{} = user} <- {:user, User.get_cached_by_id(user_id)}, + {:post, {:ok, _}} <- {:post, CommonAPI.post(user, params)} do + :ok + else error -> Logger.error( "#{__MODULE__} Couldn't create a status from the scheduled activity: #{inspect(error)}" diff --git a/test/scheduled_activity_test.exs b/test/scheduled_activity_test.exs index d2c5f5aa2..6c13d300a 100644 --- a/test/scheduled_activity_test.exs +++ b/test/scheduled_activity_test.exs @@ -102,33 +102,4 @@ test "when scheduled_at is earlier than 5 minute from now" do assert changeset.errors == [scheduled_at: {"must be at least 5 minutes from now", []}] end end - - test "creates a status from the scheduled activity" do - Pleroma.Config.put([ScheduledActivity, :enabled], true) - user = insert(:user) - - naive_datetime = - NaiveDateTime.add( - NaiveDateTime.utc_now(), - -:timer.minutes(2), - :millisecond - ) - - scheduled_activity = - insert( - :scheduled_activity, - scheduled_at: naive_datetime, - user: user, - params: %{status: "hi"} - ) - - Pleroma.Workers.ScheduledActivityWorker.perform( - %{"activity_id" => scheduled_activity.id}, - :pid - ) - - refute Repo.get(ScheduledActivity, scheduled_activity.id) - activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) - assert Pleroma.Object.normalize(activity).data["content"] == "hi" - end end diff --git a/test/workers/cron/purge_expired_activities_worker_test.exs b/test/workers/cron/purge_expired_activities_worker_test.exs index 07980bcd0..c2561683e 100644 --- a/test/workers/cron/purge_expired_activities_worker_test.exs +++ b/test/workers/cron/purge_expired_activities_worker_test.exs @@ -4,8 +4,12 @@ defmodule Pleroma.Workers.Cron.PurgeExpiredActivitiesWorkerTest do use Pleroma.DataCase + alias Pleroma.ActivityExpiration + alias Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker + import Pleroma.Factory + import ExUnit.CaptureLog clear_config([ActivityExpiration, :enabled]) @@ -31,4 +35,22 @@ test "deletes an expiration activity" do refute Pleroma.Repo.get(Pleroma.Activity, activity.id) refute Pleroma.Repo.get(Pleroma.ActivityExpiration, expiration.id) end + + describe "delete_activity/1" do + test "adds log message if activity isn't find" do + assert capture_log([level: :error], fn -> + PurgeExpiredActivitiesWorker.delete_activity(%ActivityExpiration{ + activity_id: "test-activity" + }) + end) =~ "Couldn't delete expired activity: not found activity" + end + + test "adds log message if actor isn't find" do + assert capture_log([level: :error], fn -> + PurgeExpiredActivitiesWorker.delete_activity(%ActivityExpiration{ + activity_id: "test-activity" + }) + end) =~ "Couldn't delete expired activity: not found activity" + end + end end diff --git a/test/workers/scheduled_activity_worker_test.exs b/test/workers/scheduled_activity_worker_test.exs new file mode 100644 index 000000000..1405d7142 --- /dev/null +++ b/test/workers/scheduled_activity_worker_test.exs @@ -0,0 +1,52 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.ScheduledActivityWorkerTest do + use Pleroma.DataCase + + alias Pleroma.ScheduledActivity + alias Pleroma.Workers.ScheduledActivityWorker + + import Pleroma.Factory + import ExUnit.CaptureLog + + clear_config([ScheduledActivity, :enabled]) + + test "creates a status from the scheduled activity" do + Pleroma.Config.put([ScheduledActivity, :enabled], true) + user = insert(:user) + + naive_datetime = + NaiveDateTime.add( + NaiveDateTime.utc_now(), + -:timer.minutes(2), + :millisecond + ) + + scheduled_activity = + insert( + :scheduled_activity, + scheduled_at: naive_datetime, + user: user, + params: %{status: "hi"} + ) + + ScheduledActivityWorker.perform( + %{"activity_id" => scheduled_activity.id}, + :pid + ) + + refute Repo.get(ScheduledActivity, scheduled_activity.id) + activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) + assert Pleroma.Object.normalize(activity).data["content"] == "hi" + end + + test "adds log message if ScheduledActivity isn't find" do + Pleroma.Config.put([ScheduledActivity, :enabled], true) + + assert capture_log([level: :error], fn -> + ScheduledActivityWorker.perform(%{"activity_id" => 42}, :pid) + end) =~ "Couldn't find scheduled activity" + end +end From 8589632d0905c5d028ed7f01f63a1103788d3571 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Thu, 23 Jan 2020 17:18:23 +0300 Subject: [PATCH 17/19] fixed delete ScheduledActivity --- lib/pleroma/scheduled_activity.ex | 50 ++++++++----------- .../scheduled_activity_controller.ex | 3 +- 2 files changed, 22 insertions(+), 31 deletions(-) diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index 68da0550c..e81bfcd7d 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -121,13 +121,7 @@ def create(%User{} = user, attrs) do |> Multi.insert(:scheduled_activity, new(user, attrs)) |> maybe_add_jobs(Config.get([ScheduledActivity, :enabled])) |> Repo.transaction() - |> case do - {:ok, %{scheduled_activity: scheduled_activity}} -> - {:ok, scheduled_activity} - - {:error, _, changeset, _} -> - {:error, changeset} - end + |> transaction_response end defp maybe_add_jobs(multi, true) do @@ -159,34 +153,32 @@ def update(%ScheduledActivity{id: id} = scheduled_activity, attrs) do set: [scheduled_at: get_field(changeset, :scheduled_at)] ) |> Repo.transaction() - |> case do - {:ok, %{scheduled_activity: scheduled_activity}} -> - {:ok, scheduled_activity} - - {:error, _, changeset, _} -> - {:error, changeset} - end + |> transaction_response end end - def delete_job(%ScheduledActivity{id: id} = _scheduled_activity) do - id - |> job_query - |> Repo.delete_all() - end - - def delete(%ScheduledActivity{} = scheduled_activity) do - Repo.delete(scheduled_activity) + @doc "Deletes a ScheduledActivity and linked jobs." + @spec delete(ScheduledActivity.t() | binary() | integer) :: + {:ok, ScheduledActivity.t()} | {:error, Ecto.Changeset.t()} + def delete(%ScheduledActivity{id: id} = scheduled_activity) do + Multi.new() + |> Multi.delete(:scheduled_activity, scheduled_activity, stale_error_field: :id) + |> Multi.delete_all(:jobs, job_query(id)) + |> Repo.transaction() + |> transaction_response end def delete(id) when is_binary(id) or is_integer(id) do - ScheduledActivity - |> where(id: ^id) - |> select([sa], sa) - |> Repo.delete_all() - |> case do - {1, [scheduled_activity]} -> {:ok, scheduled_activity} - _ -> :error + delete(%__MODULE__{id: id}) + end + + defp transaction_response(result) do + case result do + {:ok, %{scheduled_activity: scheduled_activity}} -> + {:ok, scheduled_activity} + + {:error, _, changeset, _} -> + {:error, changeset} end end diff --git a/lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex b/lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex index 4f9a8bdbe..ff9276541 100644 --- a/lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex @@ -45,8 +45,7 @@ def update(%{assigns: %{scheduled_activity: scheduled_activity}} = conn, params) @doc "DELETE /api/v1/scheduled_statuses/:id" def delete(%{assigns: %{scheduled_activity: scheduled_activity}} = conn, _params) do - with {:ok, scheduled_activity} <- ScheduledActivity.delete(scheduled_activity), - _ <- ScheduledActivity.delete_job(scheduled_activity) do + with {:ok, scheduled_activity} <- ScheduledActivity.delete(scheduled_activity) do render(conn, "show.json", scheduled_activity: scheduled_activity) end end From c55301e760a562ad9988911a8e2ae6b839794a08 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 10 Feb 2020 18:28:30 +0300 Subject: [PATCH 18/19] Fix a compilation error under certain circumstances I've noticed that sometimes when switching from develop to stable and back, develop fails to compile and rm -r ing the _build and deps dirs doesn't help at all. This is due to Admin API controller needing to generate JSON description of the config at compile time. Evaluating `config/description.exs` calls `Generator.list_modules_in_dir/2`, which in turn predicts the module names of files in the directory and tries to convert the predicted name to *existing* atoms. Sometimes the compiler will call that function before compiling the modules in the said directory, so the conversion will of course fail. This fixes it by removing the requirement of the atoms being existent. The function is not subjected to any untrusted user input so this should be safe. An ideal fix would be to block the compilation of docs before all modules are compiled and then get a list of compiled elixir modules under the namespace we want instead of directory hacks, but I have not been able to figure out how to do that. --- lib/pleroma/docs/generator.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/docs/generator.ex b/lib/pleroma/docs/generator.ex index 6b12dcdd9..e0fc8cd02 100644 --- a/lib/pleroma/docs/generator.ex +++ b/lib/pleroma/docs/generator.ex @@ -13,7 +13,7 @@ def list_modules_in_dir(dir, start) do |> Enum.filter(&String.ends_with?(&1, ".ex")) |> Enum.map(fn filename -> module = filename |> String.trim_trailing(".ex") |> Macro.camelize() - String.to_existing_atom(start <> module) + String.to_atom(start <> module) end) end end From a9e946b99af15a704b9008a9e032d905f5463017 Mon Sep 17 00:00:00 2001 From: Ilja Date: Mon, 10 Feb 2020 23:53:26 +0000 Subject: [PATCH 19/19] DOCS Backup/Restore/Migrate your instance * Added "Migrate" to the title because these steps can also be used to migrate the instance to another server * Added an optional step to reinstall pleroma (esp. for migrating servers) * Currently the steps threw an error 'could not execute query: ERROR: function "activity_visibility already exists with the same argument types' * I added a new step to drop and recreate an empty pleroma-database * I played around with the `-c` and `-C` options of pg_restore, but dropping and recreating seemd to be the only way I got it working * This was tested on Debian Stretch, psql (PostgreSQL) 9.6.15 --- docs/administration/backup.md | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/docs/administration/backup.md b/docs/administration/backup.md index 2c70e7bf8..685c45128 100644 --- a/docs/administration/backup.md +++ b/docs/administration/backup.md @@ -1,17 +1,35 @@ -# Backup/Restore your instance +# Backup/Restore/Move/Remove your instance ## Backup 1. Stop the Pleroma service. 2. Go to the working directory of Pleroma (default is `/opt/pleroma`) -3. Run `sudo -Hu postgres pg_dump -d --format=custom -f ` +3. Run `sudo -Hu postgres pg_dump -d --format=custom -f ` (make sure the postgres user has write access to the destination file) 4. Copy `pleroma.pgdump`, `config/prod.secret.exs` and the `uploads` folder to your backup destination. If you have other modifications, copy those changes too. 5. Restart the Pleroma service. -## Restore +## Restore/Move -1. Stop the Pleroma service. -2. Go to the working directory of Pleroma (default is `/opt/pleroma`) -3. Copy the above mentioned files back to their original position. -4. Run `sudo -Hu postgres pg_restore -d -v -1 ` -5. Restart the Pleroma service. +1. Optionally reinstall Pleroma (either on the same server or on another server if you want to move servers). Try to use the same database name. +2. Stop the Pleroma service. +3. Go to the working directory of Pleroma (default is `/opt/pleroma`) +4. Copy the above mentioned files back to their original position. +5. Drop the existing database and recreate an empty one `sudo -Hu postgres psql -c 'DROP DATABASE ;';` `sudo -Hu postgres psql -c 'CREATE DATABASE ;';` +6. Run `sudo -Hu postgres pg_restore -d -v -1 ` +7. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any. +8. Restart the Pleroma service. + +[^1]: Prefix with `MIX_ENV=prod` to run it using the production config file. + +## Remove + +1. Optionally you can remove the users of your instance. This will trigger delete requests for their accounts and posts. Note that this is 'best effort' and doesn't mean that all traces of your instance will be gone from the fediverse. + * You can do this from the admin-FE where you can select all local users and delete the accounts using the *Moderate multiple users* dropdown. + * You can also list local users and delete them individualy using the CLI tasks for [Managing users](./CLI_tasks/user.md). +2. Stop the Pleroma service `systemctl stop pleroma` +3. Disable pleroma from systemd `systemctl disable pleroma` +4. Remove the files and folders you created during installation (see installation guide). This includes the pleroma, nginx and systemd files and folders. +5. Reload nginx now that the configuration is removed `systemctl reload nginx` +6. Remove the database and database user `sudo -Hu postgres psql -c 'DROP DATABASE ;';` `sudo -Hu postgres psql -c 'DROP USER ;';` +7. Remove the system user `userdel pleroma` +8. Remove the dependencies that you don't need anymore (see installation guide). Make sure you don't remove packages that are still needed for other software that you have running!