From 0e9be6bafa81b4a55546b41e8b217fbe559cd16d Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 25 Nov 2019 14:32:20 +0300 Subject: [PATCH 01/81] moved OAuth.Token.CleanWorker to Oban Periodic jobs --- config/config.exs | 6 ++- config/description.exs | 7 ---- docs/configuration/cheatsheet.md | 13 ++++++- lib/pleroma/application.ex | 9 ----- lib/pleroma/web/oauth/token/clean_worker.ex | 38 ------------------- lib/pleroma/workers/background_worker.ex | 5 --- .../workers/cron/clear_oauth_token_worker.ex | 21 ++++++++++ mix.exs | 2 +- mix.lock | 8 ++-- 9 files changed, 41 insertions(+), 68 deletions(-) delete mode 100644 lib/pleroma/web/oauth/token/clean_worker.ex create mode 100644 lib/pleroma/workers/cron/clear_oauth_token_worker.ex diff --git a/config/config.exs b/config/config.exs index 1e36d3314..79d4ee55f 100644 --- a/config/config.exs +++ b/config/config.exs @@ -505,6 +505,9 @@ transmogrifier: 20, scheduled_activities: 10, background: 5 + ], + crontab: [ + {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker} ] config :pleroma, :workers, @@ -594,8 +597,7 @@ config :pleroma, :oauth2, token_expires_in: 600, issue_new_refresh_token: true, - clean_expired_tokens: false, - clean_expired_tokens_interval: 86_400_000 + clean_expired_tokens: false config :pleroma, :database, rum_enabled: false diff --git a/config/description.exs b/config/description.exs index 70e963399..6b89d7c67 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2251,13 +2251,6 @@ key: :clean_expired_tokens, type: :boolean, description: "Enable a background job to clean expired oauth tokens. Defaults to false" - }, - %{ - key: :clean_expired_tokens_interval, - type: :integer, - description: - "Interval to run the job to clean expired tokens. Defaults to 86_400_000 (24 hours).", - suggestions: [86_400_000] } ] }, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 07d9a1d45..e414d0b11 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -483,6 +483,7 @@ Configuration options described in [Oban readme](https://github.com/sorentwo/oba * `verbose` - logs verbosity * `prune` - non-retryable jobs [pruning settings](https://github.com/sorentwo/oban#pruning) (`:disabled` / `{:maxlen, value}` / `{:maxage, value}`) * `queues` - job queues (see below) +* `crontab` - periodic jobs, see [`Oban.Cron`](#obancron) Pleroma has the following queues: @@ -494,6 +495,12 @@ Pleroma has the following queues: * `web_push` - Web push notifications * `scheduled_activities` - Scheduled activities, see [`Pleroma.ScheduledActivity`](#pleromascheduledactivity) +#### Oban.Cron + +Pleroma has the periodic jobs: + +`Pleroma.Workers.Cron.ClearOauthTokenWorker` - the job to clean an expired oauth tokens. + Example: ```elixir @@ -504,6 +511,9 @@ config :pleroma, Oban, queues: [ federator_incoming: 50, federator_outgoing: 50 + ], + crontab: [ + {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker} ] ``` @@ -786,8 +796,7 @@ Configure OAuth 2 provider capabilities: * `token_expires_in` - The lifetime in seconds of the access token. * `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token. -* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`. -* `clean_expired_tokens_interval` - Interval to run the job to clean expired tokens. Defaults to `86_400_000` (24 hours). +* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`. Interval settings sets in configuration periodic jobs [`Oban.Cron`](#obancron) ## Link parsing diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 2b6a55f98..69a1e4a50 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -47,7 +47,6 @@ def start(_type, _args) do {Oban, Pleroma.Config.get(Oban)} ] ++ task_children(@env) ++ - oauth_cleanup_child(oauth_cleanup_enabled?()) ++ streamer_child(@env) ++ chat_child(@env, chat_enabled?()) ++ [ @@ -127,20 +126,12 @@ defp build_cachex(type, opts), defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled]) - defp oauth_cleanup_enabled?, - do: Pleroma.Config.get([:oauth2, :clean_expired_tokens], false) - defp streamer_child(:test), do: [] defp streamer_child(_) do [Pleroma.Web.Streamer.supervisor()] end - defp oauth_cleanup_child(true), - do: [Pleroma.Web.OAuth.Token.CleanWorker] - - defp oauth_cleanup_child(_), do: [] - defp chat_child(:test, _), do: [] defp chat_child(_env, true) do diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex deleted file mode 100644 index f639f9c6f..000000000 --- a/lib/pleroma/web/oauth/token/clean_worker.ex +++ /dev/null @@ -1,38 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OAuth.Token.CleanWorker do - @moduledoc """ - The module represents functions to clean an expired oauth tokens. - """ - use GenServer - - @ten_seconds 10_000 - @one_day 86_400_000 - - @interval Pleroma.Config.get( - [:oauth2, :clean_expired_tokens_interval], - @one_day - ) - - alias Pleroma.Web.OAuth.Token - alias Pleroma.Workers.BackgroundWorker - - def start_link(_), do: GenServer.start_link(__MODULE__, %{}) - - def init(_) do - Process.send_after(self(), :perform, @ten_seconds) - {:ok, nil} - end - - @doc false - def handle_info(:perform, state) do - BackgroundWorker.enqueue("clean_expired_tokens", %{}) - - Process.send_after(self(), :perform, @interval) - {:noreply, state} - end - - def perform(:clean), do: Token.delete_expired_tokens() -end diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index 7ffc8eabe..bb2b37b18 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Workers.BackgroundWorker do alias Pleroma.Activity alias Pleroma.User alias Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy - alias Pleroma.Web.OAuth.Token.CleanWorker use Pleroma.Workers.WorkerHelper, queue: "background" @@ -55,10 +54,6 @@ def perform( User.perform(:follow_import, follower, followed_identifiers) end - def perform(%{"op" => "clean_expired_tokens"}, _job) do - CleanWorker.perform(:clean) - end - def perform(%{"op" => "media_proxy_preload", "message" => message}, _job) do MediaProxyWarmingPolicy.perform(:preload, message) end diff --git a/lib/pleroma/workers/cron/clear_oauth_token_worker.ex b/lib/pleroma/workers/cron/clear_oauth_token_worker.ex new file mode 100644 index 000000000..1a4cdc198 --- /dev/null +++ b/lib/pleroma/workers/cron/clear_oauth_token_worker.ex @@ -0,0 +1,21 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.ClearOauthTokenWorker do + @moduledoc """ + The worker to clean an expired oauth tokens. + """ + + use Oban.Worker, queue: "background" + + alias Pleroma.Config + alias Pleroma.Web.OAuth.Token + + @impl Oban.Worker + def perform(_opts, _job) do + if Config.get([:oauth2, :clean_expired_tokens], false) do + Token.delete_expired_tokens() + end + end +end diff --git a/mix.exs b/mix.exs index 60c7fe3f6..31deae96e 100644 --- a/mix.exs +++ b/mix.exs @@ -102,7 +102,7 @@ defp deps do {:phoenix_ecto, "~> 4.0"}, {:ecto_sql, "~> 3.2"}, {:postgrex, ">= 0.13.5"}, - {:oban, "~> 0.8.1"}, + {:oban, "~> 0.11.1"}, {:quantum, "~> 2.3"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, diff --git a/mix.lock b/mix.lock index 3a664287c..c212b39e0 100644 --- a/mix.lock +++ b/mix.lock @@ -23,8 +23,8 @@ "decimal": {:hex, :decimal, "1.8.0", "ca462e0d885f09a1c5a342dbd7c1dcf27ea63548c65a65e67334f4b61803822e", [:mix], [], "hexpm"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, "earmark": {:hex, :earmark, "1.4.2", "3aa0bd23bc4c61cf2f1e5d752d1bb470560a6f8539974f767a38923bb20e1d7f", [:mix], [], "hexpm"}, - "ecto": {:hex, :ecto, "3.2.3", "51274df79862845b388733fddcf6f107d0c8c86e27abe7131fa98f8d30761bda", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, - "ecto_sql": {:hex, :ecto_sql, "3.2.0", "751cea597e8deb616084894dd75cbabfdbe7255ff01e8c058ca13f0353a3921b", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.2.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "ecto": {:hex, :ecto, "3.2.5", "76c864b77948a479e18e69cc1d0f0f4ee7cced1148ffe6a093ff91eba644f0b5", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "ecto_sql": {:hex, :ecto_sql, "3.2.1", "4eed4100cbb2abcff10c46660d6613693807bf64f1b865f414daccf762d3758d", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.2.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"}, "eternal": {:hex, :eternal, "1.2.1", "d5b6b2499ba876c57be2581b5b999ee9bdf861c647401066d3eeed111d096bc4", [:mix], [], "hexpm"}, "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"}, @@ -67,7 +67,7 @@ "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "ad0097e2f61d4953bfef20fb6abddf23b87111e6", [ref: "ad0097e2f61d4953bfef20fb6abddf23b87111e6", submodules: true]}, "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"}, "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]}, - "oban": {:hex, :oban, "0.8.1", "4bbf62eb1829f856d69aeb5069ac7036afe07db8221a17de2a9169cc7a58a318", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "oban": {:hex, :oban, "0.11.1", "e34964fad7f188c2c3d006485601a8897e537f7b88a31928be2833ae1cab59af", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.4", "8dd29ed783f2e12195d7e0a4640effc0a7c37e6537da491f1db01839eee6d053", [:mix], [], "hexpm"}, "phoenix": {:hex, :phoenix, "1.4.10", "619e4a545505f562cd294df52294372d012823f4fd9d34a6657a8b242898c255", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, @@ -97,7 +97,7 @@ "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.23.5", "bfd9404bbf5069b1be2ffd317923ce57e58b332e25dbca2a35dedd7820dfee5a", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, - "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, + "telemetry": {:hex, :telemetry, "0.4.1", "ae2718484892448a24470e6aa341bc847c3277bfb8d4e9289f7474d752c09c7f", [:rebar3], [], "hexpm"}, "tesla": {:hex, :tesla, "1.3.0", "f35d72f029e608f9cdc6f6d6fcc7c66cf6d6512a70cfef9206b21b8bd0203a30", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 0.4", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm"}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, From ac3abb5414bd7a5bbf53678cdf02b6f59063124c Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 26 Nov 2019 10:53:07 +0300 Subject: [PATCH 02/81] moved Pleroma.Stats to Oban Periodic jobs --- config/config.exs | 3 +- lib/pleroma/stats.ex | 39 ++++++++++++++++++------ lib/pleroma/workers/cron/stats_worker.ex | 16 ++++++++++ test/web/node_info_test.exs | 1 + 4 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 lib/pleroma/workers/cron/stats_worker.ex diff --git a/config/config.exs b/config/config.exs index 79d4ee55f..5fc92ca1b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -507,7 +507,8 @@ background: 5 ], crontab: [ - {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker} + {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, + {"0 * * * *", Pleroma.Workers.Cron.StatsWorker} ] config :pleroma, :workers, diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 8154a09b7..cf590fb01 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -9,22 +9,43 @@ defmodule Pleroma.Stats do use GenServer - @interval 1000 * 60 * 60 + @init_state %{ + peers: [], + stats: %{ + domain_count: 0, + status_count: 0, + user_count: 0 + } + } def start_link(_) do - GenServer.start_link(__MODULE__, initial_data(), name: __MODULE__) + GenServer.start_link( + __MODULE__, + @init_state, + name: __MODULE__ + ) end + @doc "Performs update stats" def force_update do GenServer.call(__MODULE__, :force_update) end + @doc "Performs collect stats" + def do_collect do + GenServer.cast(__MODULE__, :run_update) + end + + @doc "Returns stats data" + @spec get_stats() :: %{domain_count: integer(), status_count: integer(), user_count: integer()} def get_stats do %{stats: stats} = GenServer.call(__MODULE__, :get_state) stats end + @doc "Returns list peers" + @spec get_peers() :: list(String.t()) def get_peers do %{peers: peers} = GenServer.call(__MODULE__, :get_state) @@ -32,7 +53,6 @@ def get_peers do end def init(args) do - Process.send(self(), :run_update, []) {:ok, args} end @@ -45,17 +65,12 @@ def handle_call(:get_state, _from, state) do {:reply, state, state} end - def handle_info(:run_update, _state) do + def handle_cast(:run_update, _state) do new_stats = get_stat_data() - Process.send_after(self(), :run_update, @interval) {:noreply, new_stats} end - defp initial_data do - %{peers: [], stats: %{}} - end - defp get_stat_data do peers = from( @@ -74,7 +89,11 @@ defp get_stat_data do %{ peers: peers, - stats: %{domain_count: domain_count, status_count: status_count, user_count: user_count} + stats: %{ + domain_count: domain_count, + status_count: status_count, + user_count: user_count + } } end end diff --git a/lib/pleroma/workers/cron/stats_worker.ex b/lib/pleroma/workers/cron/stats_worker.ex new file mode 100644 index 000000000..425ad41ca --- /dev/null +++ b/lib/pleroma/workers/cron/stats_worker.ex @@ -0,0 +1,16 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.StatsWorker do + @moduledoc """ + The worker to update peers statistics. + """ + + use Oban.Worker, queue: "background" + + @impl Oban.Worker + def perform(_opts, _job) do + Pleroma.Stats.do_collect() + end +end diff --git a/test/web/node_info_test.exs b/test/web/node_info_test.exs index 9a574a38d..39dd72cec 100644 --- a/test/web/node_info_test.exs +++ b/test/web/node_info_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.NodeInfoTest do use Pleroma.Web.ConnCase import Pleroma.Factory + clear_config([:mrf_simple]) test "GET /.well-known/nodeinfo", %{conn: conn} do links = From 6f202a401babbc73e93fb09e697c4c844688f557 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 27 Nov 2019 09:26:37 +0300 Subject: [PATCH 03/81] moved ScheduledActivity to Oban Periodic jobs --- config/config.exs | 3 +- lib/pleroma/application.ex | 1 - .../cron/scheduled_activity_worker.ex} | 41 +++++-------------- .../workers/scheduled_activity_worker.ex | 12 ------ .../scheduled_activity_daemon_test.exs | 19 --------- test/scheduled_activity_test.exs | 28 +++++++++++++ 6 files changed, 40 insertions(+), 64 deletions(-) rename lib/pleroma/{daemons/scheduled_activity_daemon.ex => workers/cron/scheduled_activity_worker.ex} (51%) delete mode 100644 lib/pleroma/workers/scheduled_activity_worker.ex delete mode 100644 test/daemons/scheduled_activity_daemon_test.exs diff --git a/config/config.exs b/config/config.exs index 5fc92ca1b..f4f7fcce7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -508,7 +508,8 @@ ], crontab: [ {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, - {"0 * * * *", Pleroma.Workers.Cron.StatsWorker} + {"0 * * * *", Pleroma.Workers.Cron.StatsWorker}, + {"* * * * *", Pleroma.Workers.Cron.ScheduledActivityWorker} ] config :pleroma, :workers, diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 69a1e4a50..71670d9fc 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -35,7 +35,6 @@ def start(_type, _args) do Pleroma.Config.TransferTask, Pleroma.Emoji, Pleroma.Captcha, - Pleroma.Daemons.ScheduledActivityDaemon, Pleroma.Daemons.ActivityExpirationDaemon, Pleroma.Plugs.RateLimiter.Supervisor ] ++ diff --git a/lib/pleroma/daemons/scheduled_activity_daemon.ex b/lib/pleroma/workers/cron/scheduled_activity_worker.ex similarity index 51% rename from lib/pleroma/daemons/scheduled_activity_daemon.ex rename to lib/pleroma/workers/cron/scheduled_activity_worker.ex index aee5f723a..407ab687a 100644 --- a/lib/pleroma/daemons/scheduled_activity_daemon.ex +++ b/lib/pleroma/workers/cron/scheduled_activity_worker.ex @@ -2,37 +2,33 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Daemons.ScheduledActivityDaemon do +defmodule Pleroma.Workers.Cron.ScheduledActivityWorker do @moduledoc """ - Sends scheduled activities to the job queue. + The worker to post scheduled actvities. """ + use Oban.Worker, queue: "scheduled_activities" alias Pleroma.Config alias Pleroma.ScheduledActivity alias Pleroma.User alias Pleroma.Web.CommonAPI - use GenServer require Logger @schedule_interval :timer.minutes(1) - def start_link(_) do - GenServer.start_link(__MODULE__, nil) - end - - def init(_) do + @impl Oban.Worker + def perform(_opts, _job) do if Config.get([ScheduledActivity, :enabled]) do - schedule_next() - {:ok, nil} - else - :ignore + @schedule_interval + |> ScheduledActivity.due_activities() + |> Enum.each(&post_activity/1) end end - def perform(:execute, scheduled_activity_id) do + def post_activity(scheduled_activity) do try do - {:ok, scheduled_activity} = ScheduledActivity.delete(scheduled_activity_id) + {:ok, scheduled_activity} = ScheduledActivity.delete(scheduled_activity) %User{} = user = User.get_cached_by_id(scheduled_activity.user_id) {:ok, _result} = CommonAPI.post(user, scheduled_activity.params) rescue @@ -42,21 +38,4 @@ def perform(:execute, scheduled_activity_id) do ) end end - - def handle_info(:perform, state) do - ScheduledActivity.due_activities(@schedule_interval) - |> Enum.each(fn scheduled_activity -> - Pleroma.Workers.ScheduledActivityWorker.enqueue( - "execute", - %{"activity_id" => scheduled_activity.id} - ) - end) - - schedule_next() - {:noreply, state} - end - - defp schedule_next do - Process.send_after(self(), :perform, @schedule_interval) - end end diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex deleted file mode 100644 index ca7d53af1..000000000 --- a/lib/pleroma/workers/scheduled_activity_worker.ex +++ /dev/null @@ -1,12 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.ScheduledActivityWorker do - use Pleroma.Workers.WorkerHelper, queue: "scheduled_activities" - - @impl Oban.Worker - def perform(%{"op" => "execute", "activity_id" => activity_id}, _job) do - Pleroma.Daemons.ScheduledActivityDaemon.perform(:execute, activity_id) - end -end diff --git a/test/daemons/scheduled_activity_daemon_test.exs b/test/daemons/scheduled_activity_daemon_test.exs deleted file mode 100644 index c8e464491..000000000 --- a/test/daemons/scheduled_activity_daemon_test.exs +++ /dev/null @@ -1,19 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.ScheduledActivityDaemonTest do - use Pleroma.DataCase - alias Pleroma.ScheduledActivity - import Pleroma.Factory - - test "creates a status from the scheduled activity" do - user = insert(:user) - scheduled_activity = insert(:scheduled_activity, user: user, params: %{status: "hi"}) - Pleroma.Daemons.ScheduledActivityDaemon.perform(:execute, scheduled_activity.id) - - refute Repo.get(ScheduledActivity, scheduled_activity.id) - activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) - assert Pleroma.Object.normalize(activity).data["content"] == "hi" - end -end diff --git a/test/scheduled_activity_test.exs b/test/scheduled_activity_test.exs index dcf12fb49..d3d05745f 100644 --- a/test/scheduled_activity_test.exs +++ b/test/scheduled_activity_test.exs @@ -8,6 +8,8 @@ defmodule Pleroma.ScheduledActivityTest do alias Pleroma.ScheduledActivity import Pleroma.Factory + clear_config([ScheduledActivity, :enabled]) + setup context do DataCase.ensure_local_uploader(context) end @@ -61,4 +63,30 @@ test "when scheduled_at is earlier than 5 minute from now" do assert changeset.errors == [scheduled_at: {"must be at least 5 minutes from now", []}] end end + + test "creates a status from the scheduled activity" do + Pleroma.Config.put([ScheduledActivity, :enabled], true) + user = insert(:user) + + naive_datetime = + NaiveDateTime.add( + NaiveDateTime.utc_now(), + -:timer.minutes(2), + :millisecond + ) + + scheduled_activity = + insert( + :scheduled_activity, + scheduled_at: naive_datetime, + user: user, + params: %{status: "hi"} + ) + + Pleroma.Workers.Cron.ScheduledActivityWorker.perform(:opts, :pid) + + refute Repo.get(ScheduledActivity, scheduled_activity.id) + activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) + assert Pleroma.Object.normalize(activity).data["content"] == "hi" + end end From c5766a8100de465669178a98f8267425ecfe26e3 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 27 Nov 2019 13:35:02 +0300 Subject: [PATCH 04/81] moved ActivityExpiration to Oban Periodic jobs --- config/config.exs | 3 +- lib/pleroma/application.ex | 1 - .../daemons/activity_expiration_daemon.ex | 66 ------------------- .../workers/activity_expiration_worker.ex | 18 ----- .../cron/purge_expired_activities_worker.ex | 39 +++++++++++ test/activity_expiration_test.exs | 25 +++++++ .../activity_expiration_daemon_test.exs | 17 ----- 7 files changed, 66 insertions(+), 103 deletions(-) delete mode 100644 lib/pleroma/daemons/activity_expiration_daemon.ex delete mode 100644 lib/pleroma/workers/activity_expiration_worker.ex create mode 100644 lib/pleroma/workers/cron/purge_expired_activities_worker.ex delete mode 100644 test/daemons/activity_expiration_daemon_test.exs diff --git a/config/config.exs b/config/config.exs index f4f7fcce7..be8e3ee15 100644 --- a/config/config.exs +++ b/config/config.exs @@ -509,7 +509,8 @@ crontab: [ {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, {"0 * * * *", Pleroma.Workers.Cron.StatsWorker}, - {"* * * * *", Pleroma.Workers.Cron.ScheduledActivityWorker} + {"* * * * *", Pleroma.Workers.Cron.ScheduledActivityWorker}, + {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker} ] config :pleroma, :workers, diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 71670d9fc..15face96e 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -35,7 +35,6 @@ def start(_type, _args) do Pleroma.Config.TransferTask, Pleroma.Emoji, Pleroma.Captcha, - Pleroma.Daemons.ActivityExpirationDaemon, Pleroma.Plugs.RateLimiter.Supervisor ] ++ cachex_children() ++ diff --git a/lib/pleroma/daemons/activity_expiration_daemon.ex b/lib/pleroma/daemons/activity_expiration_daemon.ex deleted file mode 100644 index cab7628c4..000000000 --- a/lib/pleroma/daemons/activity_expiration_daemon.ex +++ /dev/null @@ -1,66 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Daemons.ActivityExpirationDaemon do - alias Pleroma.Activity - alias Pleroma.ActivityExpiration - alias Pleroma.Config - alias Pleroma.Repo - alias Pleroma.User - alias Pleroma.Web.CommonAPI - - require Logger - use GenServer - import Ecto.Query - - @schedule_interval :timer.minutes(1) - - def start_link(_) do - GenServer.start_link(__MODULE__, nil) - end - - @impl true - def init(_) do - if Config.get([ActivityExpiration, :enabled]) do - schedule_next() - {:ok, nil} - else - :ignore - end - end - - def perform(:execute, expiration_id) do - try do - expiration = - ActivityExpiration - |> where([e], e.id == ^expiration_id) - |> Repo.one!() - - activity = Activity.get_by_id_with_object(expiration.activity_id) - user = User.get_by_ap_id(activity.object.data["actor"]) - CommonAPI.delete(activity.id, user) - rescue - error -> - Logger.error("#{__MODULE__} Couldn't delete expired activity: #{inspect(error)}") - end - end - - @impl true - def handle_info(:perform, state) do - ActivityExpiration.due_expirations(@schedule_interval) - |> Enum.each(fn expiration -> - Pleroma.Workers.ActivityExpirationWorker.enqueue( - "activity_expiration", - %{"activity_expiration_id" => expiration.id} - ) - end) - - schedule_next() - {:noreply, state} - end - - defp schedule_next do - Process.send_after(self(), :perform, @schedule_interval) - end -end diff --git a/lib/pleroma/workers/activity_expiration_worker.ex b/lib/pleroma/workers/activity_expiration_worker.ex deleted file mode 100644 index 4e3e4195f..000000000 --- a/lib/pleroma/workers/activity_expiration_worker.ex +++ /dev/null @@ -1,18 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.ActivityExpirationWorker do - use Pleroma.Workers.WorkerHelper, queue: "activity_expiration" - - @impl Oban.Worker - def perform( - %{ - "op" => "activity_expiration", - "activity_expiration_id" => activity_expiration_id - }, - _job - ) do - Pleroma.Daemons.ActivityExpirationDaemon.perform(:execute, activity_expiration_id) - end -end diff --git a/lib/pleroma/workers/cron/purge_expired_activities_worker.ex b/lib/pleroma/workers/cron/purge_expired_activities_worker.ex new file mode 100644 index 000000000..2a7274207 --- /dev/null +++ b/lib/pleroma/workers/cron/purge_expired_activities_worker.ex @@ -0,0 +1,39 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker do + @moduledoc """ + The worker to purge expired activities. + """ + + use Oban.Worker, queue: "activity_expiration" + + alias Pleroma.Activity + alias Pleroma.ActivityExpiration + alias Pleroma.Config + alias Pleroma.User + alias Pleroma.Web.CommonAPI + + require Logger + + @interval :timer.minutes(1) + + @impl Oban.Worker + def perform(_opts, _job) do + if Config.get([ActivityExpiration, :enabled]) do + Enum.each(ActivityExpiration.due_expirations(@interval), &delete_activity/1) + end + end + + def delete_activity(expiration) do + try do + activity = Activity.get_by_id_with_object(expiration.activity_id) + user = User.get_by_ap_id(activity.object.data["actor"]) + CommonAPI.delete(activity.id, user) + rescue + error -> + Logger.error("#{__MODULE__} Couldn't delete expired activity: #{inspect(error)}") + end + end +end diff --git a/test/activity_expiration_test.exs b/test/activity_expiration_test.exs index 4948fae16..2fc593b8c 100644 --- a/test/activity_expiration_test.exs +++ b/test/activity_expiration_test.exs @@ -7,6 +7,8 @@ defmodule Pleroma.ActivityExpirationTest do alias Pleroma.ActivityExpiration import Pleroma.Factory + clear_config([ActivityExpiration, :enabled]) + test "finds activities due to be deleted only" do activity = insert(:note_activity) expiration_due = insert(:expiration_in_the_past, %{activity_id: activity.id}) @@ -24,4 +26,27 @@ test "denies expirations that don't live long enough" do now = NaiveDateTime.utc_now() assert {:error, _} = ActivityExpiration.create(activity, now) end + + test "deletes an expiration activity" do + Pleroma.Config.put([ActivityExpiration, :enabled], true) + activity = insert(:note_activity) + + naive_datetime = + NaiveDateTime.add( + NaiveDateTime.utc_now(), + -:timer.minutes(2), + :millisecond + ) + + expiration = + insert( + :expiration_in_the_past, + %{activity_id: activity.id, scheduled_at: naive_datetime} + ) + + Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker.perform(:ops, :pid) + + refute Pleroma.Repo.get(Pleroma.Activity, activity.id) + refute Pleroma.Repo.get(Pleroma.ActivityExpiration, expiration.id) + end end diff --git a/test/daemons/activity_expiration_daemon_test.exs b/test/daemons/activity_expiration_daemon_test.exs deleted file mode 100644 index b51132fb0..000000000 --- a/test/daemons/activity_expiration_daemon_test.exs +++ /dev/null @@ -1,17 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.ActivityExpirationWorkerTest do - use Pleroma.DataCase - alias Pleroma.Activity - import Pleroma.Factory - - test "deletes an activity" do - activity = insert(:note_activity) - expiration = insert(:expiration_in_the_past, %{activity_id: activity.id}) - Pleroma.Daemons.ActivityExpirationDaemon.perform(:execute, expiration.id) - - refute Repo.get(Activity, activity.id) - end -end From a4f834a687d82e7883c7dabf55b86a7e2c1dad33 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 27 Nov 2019 15:59:13 +0300 Subject: [PATCH 05/81] moved DigestEmail to Oban Periodic jobs --- config/config.exs | 18 +----- config/test.exs | 2 - lib/pleroma/application.ex | 1 - lib/pleroma/daemons/digest_email_daemon.ex | 42 -------------- lib/pleroma/scheduler.ex | 7 --- .../workers/cron/digest_emails_worker.ex | 58 +++++++++++++++++++ lib/pleroma/workers/digest_emails_worker.ex | 16 ----- mix.exs | 1 - .../cron/digest_emails_worker_test.exs} | 15 +++-- 9 files changed, 71 insertions(+), 89 deletions(-) delete mode 100644 lib/pleroma/daemons/digest_email_daemon.ex delete mode 100644 lib/pleroma/scheduler.ex create mode 100644 lib/pleroma/workers/cron/digest_emails_worker.ex delete mode 100644 lib/pleroma/workers/digest_emails_worker.ex rename test/{daemons/digest_email_daemon_test.exs => workers/cron/digest_emails_worker_test.exs} (74%) diff --git a/config/config.exs b/config/config.exs index be8e3ee15..ef9b138d1 100644 --- a/config/config.exs +++ b/config/config.exs @@ -51,20 +51,6 @@ telemetry_event: [Pleroma.Repo.Instrumenter], migration_lock: nil -scheduled_jobs = - with digest_config <- Application.get_env(:pleroma, :email_notifications)[:digest], - true <- digest_config[:active] do - [{digest_config[:schedule], {Pleroma.Daemons.DigestEmailDaemon, :perform, []}}] - else - _ -> [] - end - -config :pleroma, Pleroma.Scheduler, - global: true, - overlap: true, - timezone: :utc, - jobs: scheduled_jobs - config :pleroma, Pleroma.Captcha, enabled: false, seconds_valid: 60, @@ -510,7 +496,8 @@ {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, {"0 * * * *", Pleroma.Workers.Cron.StatsWorker}, {"* * * * *", Pleroma.Workers.Cron.ScheduledActivityWorker}, - {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker} + {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker}, + {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker} ] config :pleroma, :workers, @@ -592,7 +579,6 @@ config :pleroma, :email_notifications, digest: %{ active: false, - schedule: "0 0 * * 0", interval: 7, inactivity_threshold: 7 } diff --git a/config/test.exs b/config/test.exs index 9b737d4d7..fff709d65 100644 --- a/config/test.exs +++ b/config/test.exs @@ -68,8 +68,6 @@ queues: false, prune: :disabled -config :pleroma, Pleroma.Scheduler, jobs: [] - config :pleroma, Pleroma.ScheduledActivity, daily_user_limit: 2, total_user_limit: 3, diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 15face96e..695cc28d6 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -31,7 +31,6 @@ def start(_type, _args) do children = [ Pleroma.Repo, - Pleroma.Scheduler, Pleroma.Config.TransferTask, Pleroma.Emoji, Pleroma.Captcha, diff --git a/lib/pleroma/daemons/digest_email_daemon.ex b/lib/pleroma/daemons/digest_email_daemon.ex deleted file mode 100644 index b4c8eaad9..000000000 --- a/lib/pleroma/daemons/digest_email_daemon.ex +++ /dev/null @@ -1,42 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Daemons.DigestEmailDaemon do - alias Pleroma.Repo - alias Pleroma.Workers.DigestEmailsWorker - - import Ecto.Query - - def perform do - config = Pleroma.Config.get([:email_notifications, :digest]) - negative_interval = -Map.fetch!(config, :interval) - inactivity_threshold = Map.fetch!(config, :inactivity_threshold) - inactive_users_query = Pleroma.User.list_inactive_users_query(inactivity_threshold) - - now = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) - - from(u in inactive_users_query, - where: fragment(~s(? ->'digest' @> 'true'), u.email_notifications), - where: u.last_digest_emailed_at < datetime_add(^now, ^negative_interval, "day"), - select: u - ) - |> Repo.all() - |> Enum.each(fn user -> - DigestEmailsWorker.enqueue("digest_email", %{"user_id" => user.id}) - end) - end - - @doc """ - Send digest email to the given user. - Updates `last_digest_emailed_at` field for the user and returns the updated user. - """ - @spec perform(Pleroma.User.t()) :: Pleroma.User.t() - def perform(user) do - with %Swoosh.Email{} = email <- Pleroma.Emails.UserEmail.digest_email(user) do - Pleroma.Emails.Mailer.deliver_async(email) - end - - Pleroma.User.touch_last_digest_emailed_at(user) - end -end diff --git a/lib/pleroma/scheduler.ex b/lib/pleroma/scheduler.ex deleted file mode 100644 index d84cd99ad..000000000 --- a/lib/pleroma/scheduler.ex +++ /dev/null @@ -1,7 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Scheduler do - use Quantum.Scheduler, otp_app: :pleroma -end diff --git a/lib/pleroma/workers/cron/digest_emails_worker.ex b/lib/pleroma/workers/cron/digest_emails_worker.ex new file mode 100644 index 000000000..0a00129df --- /dev/null +++ b/lib/pleroma/workers/cron/digest_emails_worker.ex @@ -0,0 +1,58 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.DigestEmailsWorker do + @moduledoc """ + The worker to send digest emails. + """ + + use Oban.Worker, queue: "digest_emails" + + alias Pleroma.Config + alias Pleroma.Emails + alias Pleroma.Repo + alias Pleroma.User + + import Ecto.Query + + require Logger + + @impl Oban.Worker + def perform(_opts, _job) do + config = Config.get([:email_notifications, :digest]) + + if config[:active] do + negative_interval = -Map.fetch!(config, :interval) + inactivity_threshold = Map.fetch!(config, :inactivity_threshold) + inactive_users_query = User.list_inactive_users_query(inactivity_threshold) + + now = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second) + + from(u in inactive_users_query, + where: fragment(~s(? ->'digest' @> 'true'), u.email_notifications), + where: u.last_digest_emailed_at < datetime_add(^now, ^negative_interval, "day"), + select: u + ) + |> Repo.all() + |> send_emails + end + end + + def send_emails(users) do + Enum.each(users, &send_email/1) + end + + @doc """ + Send digest email to the given user. + Updates `last_digest_emailed_at` field for the user and returns the updated user. + """ + @spec send_email(User.t()) :: User.t() + def send_email(user) do + with %Swoosh.Email{} = email <- Emails.UserEmail.digest_email(user) do + Emails.Mailer.deliver_async(email) + end + + User.touch_last_digest_emailed_at(user) + end +end diff --git a/lib/pleroma/workers/digest_emails_worker.ex b/lib/pleroma/workers/digest_emails_worker.ex deleted file mode 100644 index 3e5a836d0..000000000 --- a/lib/pleroma/workers/digest_emails_worker.ex +++ /dev/null @@ -1,16 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.DigestEmailsWorker do - alias Pleroma.User - - use Pleroma.Workers.WorkerHelper, queue: "digest_emails" - - @impl Oban.Worker - def perform(%{"op" => "digest_email", "user_id" => user_id}, _job) do - user_id - |> User.get_cached_by_id() - |> Pleroma.Daemons.DigestEmailDaemon.perform() - end -end diff --git a/mix.exs b/mix.exs index 31deae96e..23d981dac 100644 --- a/mix.exs +++ b/mix.exs @@ -101,7 +101,6 @@ defp deps do {:phoenix_pubsub, "~> 1.1"}, {:phoenix_ecto, "~> 4.0"}, {:ecto_sql, "~> 3.2"}, - {:postgrex, ">= 0.13.5"}, {:oban, "~> 0.11.1"}, {:quantum, "~> 2.3"}, {:gettext, "~> 0.15"}, diff --git a/test/daemons/digest_email_daemon_test.exs b/test/workers/cron/digest_emails_worker_test.exs similarity index 74% rename from test/daemons/digest_email_daemon_test.exs rename to test/workers/cron/digest_emails_worker_test.exs index faf592d5f..073615d9e 100644 --- a/test/daemons/digest_email_daemon_test.exs +++ b/test/workers/cron/digest_emails_worker_test.exs @@ -2,16 +2,24 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.DigestEmailDaemonTest do +defmodule Pleroma.Workers.Cron.DigestEmailsWorkerTest do use Pleroma.DataCase + import Pleroma.Factory - alias Pleroma.Daemons.DigestEmailDaemon alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.CommonAPI + clear_config([:email_notifications, :digest]) + test "it sends digest emails" do + Pleroma.Config.put([:email_notifications, :digest], %{ + active: true, + inactivity_threshold: 7, + interval: 7 + }) + user = insert(:user) date = @@ -23,8 +31,7 @@ test "it sends digest emails" do {:ok, _} = User.switch_email_notifications(user2, "digest", true) CommonAPI.post(user, %{"status" => "hey @#{user2.nickname}!"}) - DigestEmailDaemon.perform() - ObanHelpers.perform_all() + Pleroma.Workers.Cron.DigestEmailsWorker.perform(:opts, :pid) # Performing job(s) enqueued at previous step ObanHelpers.perform_all() From d55859f5a4cabbe7849dc3f39079b17bd450a303 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 27 Nov 2019 16:35:12 +0300 Subject: [PATCH 06/81] remove `quantum` package --- config/config.exs | 1 - mix.exs | 3 +-- mix.lock | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/config/config.exs b/config/config.exs index ef9b138d1..54d167a82 100644 --- a/config/config.exs +++ b/config/config.exs @@ -607,7 +607,6 @@ activity_pub: nil, activity_pub_question: 30_000 -config :swarm, node_blacklist: [~r/myhtml_.*$/] # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/mix.exs b/mix.exs index 23d981dac..72e37cfe6 100644 --- a/mix.exs +++ b/mix.exs @@ -63,7 +63,7 @@ def copy_nginx_config(%{path: target_path} = release) do def application do [ mod: {Pleroma.Application, []}, - extra_applications: [:logger, :runtime_tools, :comeonin, :quack, :fast_sanitize, :swarm], + extra_applications: [:logger, :runtime_tools, :comeonin, :quack, :fast_sanitize], included_applications: [:ex_syslogger] ] end @@ -102,7 +102,6 @@ defp deps do {:phoenix_ecto, "~> 4.0"}, {:ecto_sql, "~> 3.2"}, {:oban, "~> 0.11.1"}, - {:quantum, "~> 2.3"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, {:pbkdf2_elixir, "~> 0.12.3"}, diff --git a/mix.lock b/mix.lock index c212b39e0..ff64b1d75 100644 --- a/mix.lock +++ b/mix.lock @@ -88,12 +88,10 @@ "prometheus_phoenix": {:hex, :prometheus_phoenix, "1.3.0", "c4b527e0b3a9ef1af26bdcfbfad3998f37795b9185d475ca610fe4388fdd3bb5", [:mix], [{:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm"}, "quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm"}, - "quantum": {:hex, :quantum, "2.3.4", "72a0e8855e2adc101459eac8454787cb74ab4169de6ca50f670e72142d4960e9", [:mix], [{:calendar, "~> 0.17", [hex: :calendar, repo: "hexpm", optional: true]}, {:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.12", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:swarm, "~> 3.3", [hex: :swarm, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: true]}], "hexpm"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]}, "remote_ip": {:git, "https://git.pleroma.social/pleroma/remote_ip.git", "825dc00aaba5a1b7c4202a532b696b595dd3bcb3", [ref: "825dc00aaba5a1b7c4202a532b696b595dd3bcb3"]}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.5", "6eaf7ad16cb568bb01753dbbd7a95ff8b91c7979482b95f38443fe2c8852a79b", [:make, :mix, :rebar3], [], "hexpm"}, - "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm"}, "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.23.5", "bfd9404bbf5069b1be2ffd317923ce57e58b332e25dbca2a35dedd7820dfee5a", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, From 1403a1441de36e8a58da5e996c208a9c10f65440 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Thu, 28 Nov 2019 09:02:23 +0300 Subject: [PATCH 07/81] added tests --- .../cron/clear_oauth_token_worker_test.exs | 22 +++++++++++ .../purge_expired_activities_worker_test.exs | 34 +++++++++++++++++ .../cron/scheduled_activity_worker_test.exs | 37 +++++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 test/workers/cron/clear_oauth_token_worker_test.exs create mode 100644 test/workers/cron/purge_expired_activities_worker_test.exs create mode 100644 test/workers/cron/scheduled_activity_worker_test.exs diff --git a/test/workers/cron/clear_oauth_token_worker_test.exs b/test/workers/cron/clear_oauth_token_worker_test.exs new file mode 100644 index 000000000..adea47326 --- /dev/null +++ b/test/workers/cron/clear_oauth_token_worker_test.exs @@ -0,0 +1,22 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.ClearOauthTokenWorkerTest do + use Pleroma.DataCase + + import Pleroma.Factory + alias Pleroma.Workers.Cron.ClearOauthTokenWorker + + clear_config([:oauth2, :clean_expired_tokens]) + + test "deletes expired tokens" do + insert(:oauth_token, + valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), -60 * 10) + ) + + Pleroma.Config.put([:oauth2, :clean_expired_tokens], true) + ClearOauthTokenWorker.perform(:opts, :job) + assert Pleroma.Repo.all(Pleroma.Web.OAuth.Token) == [] + end +end diff --git a/test/workers/cron/purge_expired_activities_worker_test.exs b/test/workers/cron/purge_expired_activities_worker_test.exs new file mode 100644 index 000000000..07980bcd0 --- /dev/null +++ b/test/workers/cron/purge_expired_activities_worker_test.exs @@ -0,0 +1,34 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.PurgeExpiredActivitiesWorkerTest do + use Pleroma.DataCase + alias Pleroma.ActivityExpiration + import Pleroma.Factory + + clear_config([ActivityExpiration, :enabled]) + + test "deletes an expiration activity" do + Pleroma.Config.put([ActivityExpiration, :enabled], true) + activity = insert(:note_activity) + + naive_datetime = + NaiveDateTime.add( + NaiveDateTime.utc_now(), + -:timer.minutes(2), + :millisecond + ) + + expiration = + insert( + :expiration_in_the_past, + %{activity_id: activity.id, scheduled_at: naive_datetime} + ) + + Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker.perform(:ops, :pid) + + refute Pleroma.Repo.get(Pleroma.Activity, activity.id) + refute Pleroma.Repo.get(Pleroma.ActivityExpiration, expiration.id) + end +end diff --git a/test/workers/cron/scheduled_activity_worker_test.exs b/test/workers/cron/scheduled_activity_worker_test.exs new file mode 100644 index 000000000..6f17d6f6c --- /dev/null +++ b/test/workers/cron/scheduled_activity_worker_test.exs @@ -0,0 +1,37 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.ScheduledActivityWorkerTest do + use Pleroma.DataCase + alias Pleroma.ScheduledActivity + import Pleroma.Factory + + clear_config([ScheduledActivity, :enabled]) + + test "creates a status from the scheduled activity" do + Pleroma.Config.put([ScheduledActivity, :enabled], true) + user = insert(:user) + + naive_datetime = + NaiveDateTime.add( + NaiveDateTime.utc_now(), + -:timer.minutes(2), + :millisecond + ) + + scheduled_activity = + insert( + :scheduled_activity, + scheduled_at: naive_datetime, + user: user, + params: %{status: "hi"} + ) + + Pleroma.Workers.Cron.ScheduledActivityWorker.perform(:opts, :pid) + + refute Repo.get(ScheduledActivity, scheduled_activity.id) + activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) + assert Pleroma.Object.normalize(activity).data["content"] == "hi" + end +end From 652cc6ba4b7a0494cc96ef4a9bfcaa3b5e5be60e Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 3 Dec 2019 21:30:10 +0300 Subject: [PATCH 08/81] updated ScheduledActivity --- config/config.exs | 1 - lib/pleroma/scheduled_activity.ex | 66 +++++++++++++++---- .../scheduled_activity_controller.ex | 3 +- .../controllers/status_controller.ex | 19 +++--- .../{cron => }/scheduled_activity_worker.ex | 23 ++++--- test/scheduled_activity_test.exs | 6 +- test/support/helpers.ex | 6 ++ .../scheduled_activity_controller_test.exs | 53 ++++++++++++++- .../cron/scheduled_activity_worker_test.exs | 37 ----------- 9 files changed, 142 insertions(+), 72 deletions(-) rename lib/pleroma/workers/{cron => }/scheduled_activity_worker.ex (59%) delete mode 100644 test/workers/cron/scheduled_activity_worker_test.exs diff --git a/config/config.exs b/config/config.exs index e7e17669e..259529f97 100644 --- a/config/config.exs +++ b/config/config.exs @@ -496,7 +496,6 @@ crontab: [ {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, {"0 * * * *", Pleroma.Workers.Cron.StatsWorker}, - {"* * * * *", Pleroma.Workers.Cron.ScheduledActivityWorker}, {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker}, {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker} ] diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index fea2cf3ff..96fa6a987 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -5,11 +5,13 @@ defmodule Pleroma.ScheduledActivity do use Ecto.Schema + alias Ecto.Multi alias Pleroma.Config alias Pleroma.Repo alias Pleroma.ScheduledActivity alias Pleroma.User alias Pleroma.Web.CommonAPI.Utils + alias Pleroma.Workers.ScheduledActivityWorker import Ecto.Query import Ecto.Changeset @@ -105,14 +107,29 @@ def far_enough?(scheduled_at) do end def new(%User{} = user, attrs) do - %ScheduledActivity{user_id: user.id} - |> changeset(attrs) + changeset(%ScheduledActivity{user_id: user.id}, attrs) end + @doc """ + Creates ScheduledActivity and add to queue to perform at scheduled_at date + """ + @spec create(User.t(), map()) :: {:ok, ScheduledActivity.t()} | {:error, Ecto.Changeset.t()} def create(%User{} = user, attrs) do - user - |> new(attrs) - |> Repo.insert() + Multi.new() + |> Multi.insert(:scheduled_activity, new(user, attrs)) + |> Multi.run(:scheduled_activity_job, fn _repo, %{scheduled_activity: activity} -> + %{activity_id: activity.id} + |> ScheduledActivityWorker.new(scheduled_at: activity.scheduled_at) + |> Oban.insert() + end) + |> Repo.transaction() + |> case do + {:ok, %{scheduled_activity: scheduled_activity}} -> + {:ok, scheduled_activity} + + {:error, _, changeset, _} -> + {:error, changeset} + end end def get(%User{} = user, scheduled_activity_id) do @@ -122,15 +139,35 @@ def get(%User{} = user, scheduled_activity_id) do |> Repo.one() end - def update(%ScheduledActivity{} = scheduled_activity, attrs) do - scheduled_activity - |> update_changeset(attrs) - |> Repo.update() + @spec update(ScheduledActivity.t(), map()) :: + {:ok, ScheduledActivity.t()} | {:error, Ecto.Changeset.t()} + def update(%ScheduledActivity{id: id} = scheduled_activity, attrs) do + with {:error, %Ecto.Changeset{valid?: true} = changeset} <- + {:error, update_changeset(scheduled_activity, attrs)} do + Multi.new() + |> Multi.update(:scheduled_activity, changeset) + |> Multi.update_all(:scheduled_job, job_query(id), + set: [scheduled_at: changeset.changes[:scheduled_at]] + ) + |> Repo.transaction() + |> case do + {:ok, %{scheduled_activity: scheduled_activity}} -> + {:ok, scheduled_activity} + + {:error, _, changeset, _} -> + {:error, changeset} + end + end + end + + def delete_job(%ScheduledActivity{id: id} = _scheduled_activity) do + id + |> job_query + |> Repo.delete_all() end def delete(%ScheduledActivity{} = scheduled_activity) do - scheduled_activity - |> Repo.delete() + Repo.delete(scheduled_activity) end def delete(id) when is_binary(id) or is_integer(id) do @@ -158,4 +195,11 @@ def due_activities(offset \\ 0) do |> where([sa], sa.scheduled_at < ^naive_datetime) |> Repo.all() end + + def job_query(scheduled_activity_id) do + from(j in Oban.Job, + where: j.queue == "scheduled_activities", + where: fragment("args ->> 'activity_id' = ?::text", ^to_string(scheduled_activity_id)) + ) + end end diff --git a/lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex b/lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex index ff9276541..4f9a8bdbe 100644 --- a/lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex @@ -45,7 +45,8 @@ def update(%{assigns: %{scheduled_activity: scheduled_activity}} = conn, params) @doc "DELETE /api/v1/scheduled_statuses/:id" def delete(%{assigns: %{scheduled_activity: scheduled_activity}} = conn, _params) do - with {:ok, scheduled_activity} <- ScheduledActivity.delete(scheduled_activity) do + with {:ok, scheduled_activity} <- ScheduledActivity.delete(scheduled_activity), + _ <- ScheduledActivity.delete_job(scheduled_activity) do render(conn, "show.json", scheduled_activity: scheduled_activity) end end diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 74b223cf4..d70749dfa 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -124,15 +124,18 @@ def create( ) do params = Map.put(params, "in_reply_to_status_id", params["in_reply_to_id"]) - if ScheduledActivity.far_enough?(scheduled_at) do - with {:ok, scheduled_activity} <- - ScheduledActivity.create(user, %{"params" => params, "scheduled_at" => scheduled_at}) do - conn - |> put_view(ScheduledActivityView) - |> render("show.json", scheduled_activity: scheduled_activity) - end + with {:far_enough, true} <- {:far_enough, ScheduledActivity.far_enough?(scheduled_at)}, + attrs <- %{"params" => params, "scheduled_at" => scheduled_at}, + {:ok, scheduled_activity} <- ScheduledActivity.create(user, attrs) do + conn + |> put_view(ScheduledActivityView) + |> render("show.json", scheduled_activity: scheduled_activity) else - create(conn, Map.drop(params, ["scheduled_at"])) + {:far_enough, _} -> + create(conn, Map.drop(params, ["scheduled_at"])) + + error -> + error end end diff --git a/lib/pleroma/workers/cron/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex similarity index 59% rename from lib/pleroma/workers/cron/scheduled_activity_worker.ex rename to lib/pleroma/workers/scheduled_activity_worker.ex index 407ab687a..5109d7f75 100644 --- a/lib/pleroma/workers/cron/scheduled_activity_worker.ex +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -2,12 +2,13 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Workers.Cron.ScheduledActivityWorker do +defmodule Pleroma.Workers.ScheduledActivityWorker do @moduledoc """ - The worker to post scheduled actvities. + The worker to post scheduled activity. """ - use Oban.Worker, queue: "scheduled_activities" + use Pleroma.Workers.WorkerHelper, queue: "scheduled_activities" + alias Pleroma.Config alias Pleroma.ScheduledActivity alias Pleroma.User @@ -15,18 +16,20 @@ defmodule Pleroma.Workers.Cron.ScheduledActivityWorker do require Logger - @schedule_interval :timer.minutes(1) - @impl Oban.Worker - def perform(_opts, _job) do + def perform(%{"activity_id" => activity_id}, _job) do if Config.get([ScheduledActivity, :enabled]) do - @schedule_interval - |> ScheduledActivity.due_activities() - |> Enum.each(&post_activity/1) + case Pleroma.Repo.get(ScheduledActivity, activity_id) do + %ScheduledActivity{} = scheduled_activity -> + post_activity(scheduled_activity) + + _ -> + Logger.error("#{__MODULE__} Couldn't find scheduled activity: #{activity_id}") + end end end - def post_activity(scheduled_activity) do + defp post_activity(%ScheduledActivity{} = scheduled_activity) do try do {:ok, scheduled_activity} = ScheduledActivity.delete(scheduled_activity) %User{} = user = User.get_cached_by_id(scheduled_activity.user_id) diff --git a/test/scheduled_activity_test.exs b/test/scheduled_activity_test.exs index d3d05745f..663cfdd34 100644 --- a/test/scheduled_activity_test.exs +++ b/test/scheduled_activity_test.exs @@ -26,6 +26,7 @@ test "when daily user limit is exceeded" do attrs = %{params: %{}, scheduled_at: today} {:ok, _} = ScheduledActivity.create(user, attrs) {:ok, _} = ScheduledActivity.create(user, attrs) + {:error, changeset} = ScheduledActivity.create(user, attrs) assert changeset.errors == [scheduled_at: {"daily limit exceeded", []}] end @@ -83,7 +84,10 @@ test "creates a status from the scheduled activity" do params: %{status: "hi"} ) - Pleroma.Workers.Cron.ScheduledActivityWorker.perform(:opts, :pid) + Pleroma.Workers.ScheduledActivityWorker.perform( + %{"activity_id" => scheduled_activity.id}, + :pid + ) refute Repo.get(ScheduledActivity, scheduled_activity.id) activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) diff --git a/test/support/helpers.ex b/test/support/helpers.ex index ce39dd9d8..ec556a916 100644 --- a/test/support/helpers.ex +++ b/test/support/helpers.ex @@ -53,6 +53,12 @@ defmacro __using__(_opts) do clear_config_all: 2 ] + def to_datetime(naive_datetime) do + naive_datetime + |> DateTime.from_naive!("Etc/UTC") + |> DateTime.truncate(:second) + end + def collect_ids(collection) do collection |> Enum.map(& &1.id) diff --git a/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs b/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs index ae5fee2bc..5f3a376be 100644 --- a/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs +++ b/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs @@ -9,6 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do alias Pleroma.ScheduledActivity import Pleroma.Factory + import Ecto.Query test "shows scheduled activities", %{conn: conn} do user = insert(:user) @@ -68,7 +69,30 @@ test "shows a scheduled activity", %{conn: conn} do test "updates a scheduled activity", %{conn: conn} do user = insert(:user) - scheduled_activity = insert(:scheduled_activity, user: user) + + scheduled_at = + NaiveDateTime.add( + NaiveDateTime.utc_now(), + :timer.minutes(60), + :millisecond + ) + + {:ok, scheduled_activity} = + ScheduledActivity.create( + user, + %{ + scheduled_at: scheduled_at, + params: build(:note).data + } + ) + + scheduled_activity_job = + Repo.one(from(j in Oban.Job, where: j.queue == "scheduled_activities")) + + assert scheduled_activity_job.args == %{"activity_id" => scheduled_activity.id} + + assert DateTime.truncate(scheduled_activity_job.scheduled_at, :second) == + to_datetime(scheduled_at) new_scheduled_at = NaiveDateTime.add(NaiveDateTime.utc_now(), :timer.minutes(120), :millisecond) @@ -82,6 +106,10 @@ test "updates a scheduled activity", %{conn: conn} do assert %{"scheduled_at" => expected_scheduled_at} = json_response(res_conn, 200) assert expected_scheduled_at == Pleroma.Web.CommonAPI.Utils.to_masto_date(new_scheduled_at) + scheduled_activity_job = refresh_record(scheduled_activity_job) + + assert DateTime.truncate(scheduled_activity_job.scheduled_at, :second) == + to_datetime(new_scheduled_at) res_conn = conn @@ -93,7 +121,25 @@ test "updates a scheduled activity", %{conn: conn} do test "deletes a scheduled activity", %{conn: conn} do user = insert(:user) - scheduled_activity = insert(:scheduled_activity, user: user) + + {:ok, scheduled_activity} = + ScheduledActivity.create( + user, + %{ + scheduled_at: + NaiveDateTime.add( + NaiveDateTime.utc_now(), + :timer.minutes(60), + :millisecond + ), + params: build(:note).data + } + ) + + scheduled_activity_job = + Repo.one(from(j in Oban.Job, where: j.queue == "scheduled_activities")) + + assert scheduled_activity_job.args == %{"activity_id" => scheduled_activity.id} res_conn = conn @@ -101,7 +147,8 @@ test "deletes a scheduled activity", %{conn: conn} do |> delete("/api/v1/scheduled_statuses/#{scheduled_activity.id}") assert %{} = json_response(res_conn, 200) - assert nil == Repo.get(ScheduledActivity, scheduled_activity.id) + refute Repo.get(ScheduledActivity, scheduled_activity.id) + refute Repo.get(Oban.Job, scheduled_activity_job.id) res_conn = conn diff --git a/test/workers/cron/scheduled_activity_worker_test.exs b/test/workers/cron/scheduled_activity_worker_test.exs deleted file mode 100644 index 6f17d6f6c..000000000 --- a/test/workers/cron/scheduled_activity_worker_test.exs +++ /dev/null @@ -1,37 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.Cron.ScheduledActivityWorkerTest do - use Pleroma.DataCase - alias Pleroma.ScheduledActivity - import Pleroma.Factory - - clear_config([ScheduledActivity, :enabled]) - - test "creates a status from the scheduled activity" do - Pleroma.Config.put([ScheduledActivity, :enabled], true) - user = insert(:user) - - naive_datetime = - NaiveDateTime.add( - NaiveDateTime.utc_now(), - -:timer.minutes(2), - :millisecond - ) - - scheduled_activity = - insert( - :scheduled_activity, - scheduled_at: naive_datetime, - user: user, - params: %{status: "hi"} - ) - - Pleroma.Workers.Cron.ScheduledActivityWorker.perform(:opts, :pid) - - refute Repo.get(ScheduledActivity, scheduled_activity.id) - activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) - assert Pleroma.Object.normalize(activity).data["content"] == "hi" - end -end From 469c0956cd937b28a38f8c982b670207ec14b843 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 4 Dec 2019 09:43:01 +0300 Subject: [PATCH 09/81] update excoveralls --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 618864eff..fc84ae962 100644 --- a/mix.exs +++ b/mix.exs @@ -156,7 +156,7 @@ defp deps do {:esshd, "~> 0.1.0", runtime: Application.get_env(:esshd, :enabled, false)}, {:ex_const, "~> 0.2"}, {:plug_static_index_html, "~> 1.0.0"}, - {:excoveralls, "~> 0.11.1", only: :test}, + {:excoveralls, "~> 0.12.1", only: :test}, {:flake_id, "~> 0.1.0"}, {:remote_ip, git: "https://git.pleroma.social/pleroma/remote_ip.git", diff --git a/mix.lock b/mix.lock index 2e12d9b73..c76874a4d 100644 --- a/mix.lock +++ b/mix.lock @@ -34,7 +34,7 @@ "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, "ex_machina": {:hex, :ex_machina, "2.3.0", "92a5ad0a8b10ea6314b876a99c8c9e3f25f4dde71a2a835845b136b9adaf199a", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm"}, "ex_syslogger": {:git, "https://github.com/slashmili/ex_syslogger.git", "f3963399047af17e038897c69e20d552e6899e1d", [tag: "1.4.0"]}, - "excoveralls": {:hex, :excoveralls, "0.11.2", "0c6f2c8db7683b0caa9d490fb8125709c54580b4255ffa7ad35f3264b075a643", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, + "excoveralls": {:hex, :excoveralls, "0.12.1", "a553c59f6850d0aff3770e4729515762ba7c8e41eedde03208182a8dc9d0ce07", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, "fast_html": {:hex, :fast_html, "0.99.4", "d80812664f0429607e1d880fba0ef04da87a2e4fa596701bcaae17953535695c", [:make, :mix], [], "hexpm"}, "fast_sanitize": {:hex, :fast_sanitize, "0.1.4", "6c2e7203ca2f8275527a3021ba6e9d5d4ee213a47dc214a97c128737c9e56df1", [:mix], [{:fast_html, "~> 0.99", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, From 3a0a400fe154f44ab37b79eeb3a07f919517743e Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 4 Dec 2019 09:53:01 +0300 Subject: [PATCH 10/81] add @type to ScheduledActivity --- lib/pleroma/scheduled_activity.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index 96fa6a987..ea78dce6a 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -16,6 +16,8 @@ defmodule Pleroma.ScheduledActivity do import Ecto.Query import Ecto.Changeset + @type t :: %__MODULE__{} + @min_offset :timer.minutes(5) schema "scheduled_activities" do From c0d572d0bf842fae08c609aa58c82554bee3a263 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 4 Dec 2019 17:28:57 +0300 Subject: [PATCH 11/81] added test --- test/scheduled_activity_test.exs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/scheduled_activity_test.exs b/test/scheduled_activity_test.exs index 663cfdd34..b367ae5fb 100644 --- a/test/scheduled_activity_test.exs +++ b/test/scheduled_activity_test.exs @@ -24,9 +24,13 @@ test "when daily user limit is exceeded" do |> NaiveDateTime.to_iso8601() attrs = %{params: %{}, scheduled_at: today} - {:ok, _} = ScheduledActivity.create(user, attrs) - {:ok, _} = ScheduledActivity.create(user, attrs) + {:ok, sa1} = ScheduledActivity.create(user, attrs) + {:ok, sa2} = ScheduledActivity.create(user, attrs) + jobs = + Repo.all(from(j in Oban.Job, where: j.queue == "scheduled_activities", select: j.args)) + + assert jobs == [%{"activity_id" => sa1.id}, %{"activity_id" => sa2.id}] {:error, changeset} = ScheduledActivity.create(user, attrs) assert changeset.errors == [scheduled_at: {"daily limit exceeded", []}] end From 3c3bba0b7c65187b3270ef3402442cf870a55198 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 4 Dec 2019 21:18:05 +0300 Subject: [PATCH 12/81] fix ScheduledActivity --- lib/pleroma/scheduled_activity.ex | 17 ++++++--- test/scheduled_activity_test.exs | 36 ++++++++++++++++++- .../scheduled_activity_controller_test.exs | 4 +++ 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index ea78dce6a..d01100702 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -119,11 +119,7 @@ def new(%User{} = user, attrs) do def create(%User{} = user, attrs) do Multi.new() |> Multi.insert(:scheduled_activity, new(user, attrs)) - |> Multi.run(:scheduled_activity_job, fn _repo, %{scheduled_activity: activity} -> - %{activity_id: activity.id} - |> ScheduledActivityWorker.new(scheduled_at: activity.scheduled_at) - |> Oban.insert() - end) + |> maybe_add_jobs(Config.get([ScheduledActivity, :enabled])) |> Repo.transaction() |> case do {:ok, %{scheduled_activity: scheduled_activity}} -> @@ -134,6 +130,17 @@ def create(%User{} = user, attrs) do end end + defp maybe_add_jobs(multi, true) do + multi + |> Multi.run(:scheduled_activity_job, fn _repo, %{scheduled_activity: activity} -> + %{activity_id: activity.id} + |> ScheduledActivityWorker.new(scheduled_at: activity.scheduled_at) + |> Oban.insert() + end) + end + + defp maybe_add_jobs(multi, _), do: multi + def get(%User{} = user, scheduled_activity_id) do ScheduledActivity |> where(user_id: ^user.id) diff --git a/test/scheduled_activity_test.exs b/test/scheduled_activity_test.exs index b367ae5fb..d2c5f5aa2 100644 --- a/test/scheduled_activity_test.exs +++ b/test/scheduled_activity_test.exs @@ -15,7 +15,8 @@ defmodule Pleroma.ScheduledActivityTest do end describe "creation" do - test "when daily user limit is exceeded" do + test "scheduled activities with jobs when ScheduledActivity enabled" do + Pleroma.Config.put([ScheduledActivity, :enabled], true) user = insert(:user) today = @@ -31,6 +32,39 @@ test "when daily user limit is exceeded" do Repo.all(from(j in Oban.Job, where: j.queue == "scheduled_activities", select: j.args)) assert jobs == [%{"activity_id" => sa1.id}, %{"activity_id" => sa2.id}] + end + + test "scheduled activities without jobs when ScheduledActivity disabled" do + Pleroma.Config.put([ScheduledActivity, :enabled], false) + user = insert(:user) + + today = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(6), :millisecond) + |> NaiveDateTime.to_iso8601() + + attrs = %{params: %{}, scheduled_at: today} + {:ok, _sa1} = ScheduledActivity.create(user, attrs) + {:ok, _sa2} = ScheduledActivity.create(user, attrs) + + jobs = + Repo.all(from(j in Oban.Job, where: j.queue == "scheduled_activities", select: j.args)) + + assert jobs == [] + end + + test "when daily user limit is exceeded" do + user = insert(:user) + + today = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(6), :millisecond) + |> NaiveDateTime.to_iso8601() + + attrs = %{params: %{}, scheduled_at: today} + {:ok, _} = ScheduledActivity.create(user, attrs) + {:ok, _} = ScheduledActivity.create(user, attrs) + {:error, changeset} = ScheduledActivity.create(user, attrs) assert changeset.errors == [scheduled_at: {"daily limit exceeded", []}] end diff --git a/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs b/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs index 5f3a376be..478631a12 100644 --- a/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs +++ b/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs @@ -11,6 +11,8 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do import Pleroma.Factory import Ecto.Query + clear_config([ScheduledActivity, :enabled]) + test "shows scheduled activities", %{conn: conn} do user = insert(:user) scheduled_activity_id1 = insert(:scheduled_activity, user: user).id |> to_string() @@ -68,6 +70,7 @@ test "shows a scheduled activity", %{conn: conn} do end test "updates a scheduled activity", %{conn: conn} do + Pleroma.Config.put([ScheduledActivity, :enabled], true) user = insert(:user) scheduled_at = @@ -120,6 +123,7 @@ test "updates a scheduled activity", %{conn: conn} do end test "deletes a scheduled activity", %{conn: conn} do + Pleroma.Config.put([ScheduledActivity, :enabled], true) user = insert(:user) {:ok, scheduled_activity} = From a12b6454bb0a270732f9b55f8d4366c9add44136 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 16 Dec 2019 22:24:03 +0700 Subject: [PATCH 13/81] Add an option to require fetches to be signed --- CHANGELOG.md | 1 + config/config.exs | 3 +- docs/configuration/cheatsheet.md | 9 ++-- lib/pleroma/plugs/http_signature.ex | 43 ++++++++++++------ test/plugs/http_signature_plug_test.exs | 58 +++++++++++++++++++++++++ 5 files changed, 95 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c133cd9ec..ee9e1091c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - User notification settings: Add `privacy_option` option. - User settings: Add _This account is a_ option. - OAuth: admin scopes support (relevant setting: `[:auth, :enforce_oauth_admin_scope_usage]`). +- Add an option `authorized_fetch_mode` to requrie HTTP Signature for AP fetches.
API Changes diff --git a/config/config.exs b/config/config.exs index 370ddd855..541fcc2d4 100644 --- a/config/config.exs +++ b/config/config.exs @@ -343,7 +343,8 @@ unfollow_blocked: true, outgoing_blocks: true, follow_handshake_timeout: 500, - sign_object_fetches: true + sign_object_fetches: true, + authorized_fetch_mode: false config :pleroma, :streamer, workers: 3, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index ce2a14210..8fa4a1747 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -147,10 +147,11 @@ config :pleroma, :mrf_user_allowlist, * `:reject` rejects the message entirely ### :activitypub -* ``unfollow_blocked``: Whether blocks result in people getting unfollowed -* ``outgoing_blocks``: Whether to federate blocks to other instances -* ``deny_follow_blocked``: Whether to disallow following an account that has blocked the user in question -* ``sign_object_fetches``: Sign object fetches with HTTP signatures +* `unfollow_blocked`: Whether blocks result in people getting unfollowed +* `outgoing_blocks`: Whether to federate blocks to other instances +* `deny_follow_blocked`: Whether to disallow following an account that has blocked the user in question +* `sign_object_fetches`: Sign object fetches with HTTP signatures +* `authorized_fetch_mode`: Require HTTP Signature for AP fetches ### :fetch_initial_posts * `enabled`: if enabled, when a new user is federated with, fetch some of their latest posts diff --git a/lib/pleroma/plugs/http_signature.ex b/lib/pleroma/plugs/http_signature.ex index 23d22a712..ecd7a55bf 100644 --- a/lib/pleroma/plugs/http_signature.ex +++ b/lib/pleroma/plugs/http_signature.ex @@ -15,25 +15,23 @@ def call(%{assigns: %{valid_signature: true}} = conn, _opts) do end def call(conn, _opts) do - headers = get_req_header(conn, "signature") - signature = Enum.at(headers, 0) + conn + |> maybe_assign_valid_signature() + |> maybe_require_signature() + end - if signature do + defp maybe_assign_valid_signature(conn) do + if has_signature_header?(conn) do # set (request-target) header to the appropriate value # we also replace the digest header with the one we computed - conn = - conn - |> put_req_header( - "(request-target)", - String.downcase("#{conn.method}") <> " #{conn.request_path}" - ) + request_target = String.downcase("#{conn.method}") <> " #{conn.request_path}" conn = - if conn.assigns[:digest] do - conn - |> put_req_header("digest", conn.assigns[:digest]) - else - conn + conn + |> put_req_header("(request-target)", request_target) + |> case do + %{assigns: %{digest: digest}} = conn -> put_req_header(conn, "digest", digest) + conn -> conn end assign(conn, :valid_signature, HTTPSignatures.validate_conn(conn)) @@ -42,4 +40,21 @@ def call(conn, _opts) do conn end end + + defp has_signature_header?(conn) do + conn |> get_req_header("signature") |> Enum.at(0, false) + end + + defp maybe_require_signature(%{assigns: %{valid_signature: true}} = conn), do: conn + + defp maybe_require_signature(conn) do + if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do + conn + |> put_status(:unauthorized) + |> Phoenix.Controller.text("Request not signed") + |> halt() + else + conn + end + end end diff --git a/test/plugs/http_signature_plug_test.exs b/test/plugs/http_signature_plug_test.exs index d8ace36da..007193dd9 100644 --- a/test/plugs/http_signature_plug_test.exs +++ b/test/plugs/http_signature_plug_test.exs @@ -23,7 +23,65 @@ test "it call HTTPSignatures to check validity if the actor sighed it" do |> HTTPSignaturePlug.call(%{}) assert conn.assigns.valid_signature == true + assert conn.halted == false assert called(HTTPSignatures.validate_conn(:_)) end end + + describe "requries a signature when `authorized_fetch_mode` is enabled" do + setup do + Pleroma.Config.put([:activitypub, :authorized_fetch_mode], true) + + on_exit(fn -> + Pleroma.Config.put([:activitypub, :authorized_fetch_mode], false) + end) + + params = %{"actor" => "http://mastodon.example.org/users/admin"} + conn = build_conn(:get, "/doesntmattter", params) + + [conn: conn] + end + + test "when signature header is present", %{conn: conn} do + with_mock HTTPSignatures, validate_conn: fn _ -> false end do + conn = + conn + |> put_req_header( + "signature", + "keyId=\"http://mastodon.example.org/users/admin#main-key" + ) + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == false + assert conn.halted == true + assert conn.status == 401 + assert conn.state == :sent + assert conn.resp_body == "Request not signed" + assert called(HTTPSignatures.validate_conn(:_)) + end + + with_mock HTTPSignatures, validate_conn: fn _ -> true end do + conn = + conn + |> put_req_header( + "signature", + "keyId=\"http://mastodon.example.org/users/admin#main-key" + ) + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == true + assert conn.halted == false + assert called(HTTPSignatures.validate_conn(:_)) + end + end + + test "halts the connection when `signature` header is not present", %{conn: conn} do + conn = HTTPSignaturePlug.call(conn, %{}) + assert conn.assigns[:valid_signature] == nil + assert conn.halted == true + assert conn.status == 401 + assert conn.state == :sent + assert conn.resp_body == "Request not signed" + end + end end From e1fa8c11a9ea26f54a231cbdacdc8befe634b57e Mon Sep 17 00:00:00 2001 From: minibikini Date: Mon, 16 Dec 2019 18:39:59 +0000 Subject: [PATCH 14/81] Apply suggestion to test/plugs/http_signature_plug_test.exs --- test/plugs/http_signature_plug_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/plugs/http_signature_plug_test.exs b/test/plugs/http_signature_plug_test.exs index 007193dd9..77e790288 100644 --- a/test/plugs/http_signature_plug_test.exs +++ b/test/plugs/http_signature_plug_test.exs @@ -28,7 +28,7 @@ test "it call HTTPSignatures to check validity if the actor sighed it" do end end - describe "requries a signature when `authorized_fetch_mode` is enabled" do + describe "requires a signature when `authorized_fetch_mode` is enabled" do setup do Pleroma.Config.put([:activitypub, :authorized_fetch_mode], true) From d81c3afbb2f020226909afed851d2aa623fdcdb0 Mon Sep 17 00:00:00 2001 From: minibikini Date: Mon, 16 Dec 2019 18:40:12 +0000 Subject: [PATCH 15/81] Apply suggestion to CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee9e1091c..3c2b49962 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,7 +53,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - User notification settings: Add `privacy_option` option. - User settings: Add _This account is a_ option. - OAuth: admin scopes support (relevant setting: `[:auth, :enforce_oauth_admin_scope_usage]`). -- Add an option `authorized_fetch_mode` to requrie HTTP Signature for AP fetches. +- Add an option `authorized_fetch_mode` to require HTTP Signatures for AP fetches.
API Changes From 36d66d965519037d086ad5080ccf833801c3381e Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 17 Dec 2019 02:06:58 +0700 Subject: [PATCH 16/81] Fix typo --- CHANGELOG.md | 2 +- docs/configuration/cheatsheet.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c2b49962..93c7485aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,7 +53,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - User notification settings: Add `privacy_option` option. - User settings: Add _This account is a_ option. - OAuth: admin scopes support (relevant setting: `[:auth, :enforce_oauth_admin_scope_usage]`). -- Add an option `authorized_fetch_mode` to require HTTP Signatures for AP fetches. +- Add an option `authorized_fetch_mode` to require HTTP signatures for AP fetches.
API Changes diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 8fa4a1747..cdc7c5ee0 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -151,7 +151,7 @@ config :pleroma, :mrf_user_allowlist, * `outgoing_blocks`: Whether to federate blocks to other instances * `deny_follow_blocked`: Whether to disallow following an account that has blocked the user in question * `sign_object_fetches`: Sign object fetches with HTTP signatures -* `authorized_fetch_mode`: Require HTTP Signature for AP fetches +* `authorized_fetch_mode`: Require HTTP signatures for AP fetches ### :fetch_initial_posts * `enabled`: if enabled, when a new user is federated with, fetch some of their latest posts From 775212121cc3eb108bca6c4b94a3fdf6d8d8fcd1 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 19 Dec 2019 20:17:18 +0700 Subject: [PATCH 17/81] Verify HTTP signatures only when request accepts "activity+json" type --- lib/pleroma/plugs/http_signature.ex | 13 +++++++++---- test/plugs/http_signature_plug_test.exs | 4 +++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/plugs/http_signature.ex b/lib/pleroma/plugs/http_signature.ex index ecd7a55bf..477a5b578 100644 --- a/lib/pleroma/plugs/http_signature.ex +++ b/lib/pleroma/plugs/http_signature.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do import Plug.Conn + import Phoenix.Controller, only: [get_format: 1, text: 2] require Logger def init(options) do @@ -15,9 +16,13 @@ def call(%{assigns: %{valid_signature: true}} = conn, _opts) do end def call(conn, _opts) do - conn - |> maybe_assign_valid_signature() - |> maybe_require_signature() + if get_format(conn) == "activity+json" do + conn + |> maybe_assign_valid_signature() + |> maybe_require_signature() + else + conn + end end defp maybe_assign_valid_signature(conn) do @@ -51,7 +56,7 @@ defp maybe_require_signature(conn) do if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do conn |> put_status(:unauthorized) - |> Phoenix.Controller.text("Request not signed") + |> text("Request not signed") |> halt() else conn diff --git a/test/plugs/http_signature_plug_test.exs b/test/plugs/http_signature_plug_test.exs index 77e790288..55e8bafc0 100644 --- a/test/plugs/http_signature_plug_test.exs +++ b/test/plugs/http_signature_plug_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do alias Pleroma.Web.Plugs.HTTPSignaturePlug import Plug.Conn + import Phoenix.Controller, only: [put_format: 2] import Mock test "it call HTTPSignatures to check validity if the actor sighed it" do @@ -20,6 +21,7 @@ test "it call HTTPSignatures to check validity if the actor sighed it" do "signature", "keyId=\"http://mastodon.example.org/users/admin#main-key" ) + |> put_format("activity+json") |> HTTPSignaturePlug.call(%{}) assert conn.assigns.valid_signature == true @@ -37,7 +39,7 @@ test "it call HTTPSignatures to check validity if the actor sighed it" do end) params = %{"actor" => "http://mastodon.example.org/users/admin"} - conn = build_conn(:get, "/doesntmattter", params) + conn = build_conn(:get, "/doesntmattter", params) |> put_format("activity+json") [conn: conn] end From e02eb8de3ec8130e9808c6a54a5c78f7ec8feb23 Mon Sep 17 00:00:00 2001 From: ilja Date: Wed, 15 Jan 2020 09:12:24 +0100 Subject: [PATCH 18/81] Add OTP to updating.md * I removed the Update section from the OTP install and added it to updating.md * I also added a link to the updating.md from the OTP install page * I added a Questions section to the OTP install, similar to the debian_based_en.md * Restructured the updating.md a bit I would very much also like to link to the changelog, but I don't know how to do it because I can't find the page in the docs? --- docs/administration/updating.md | 17 +++++++++++++++++ docs/installation/otp_en.md | 17 ++++++----------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/docs/administration/updating.md b/docs/administration/updating.md index 84e6ef18d..2a08dac1f 100644 --- a/docs/administration/updating.md +++ b/docs/administration/updating.md @@ -1,4 +1,21 @@ # Updating your instance + +You should **always check the release notes/changelog** in case there are config deprecations, special update special update steps, etc. + +Besides that, doing the following is generally enough: + +## For OTP installations + +```sh +# Download the new release +su pleroma -s $SHELL -lc "./bin/pleroma_ctl update" + +# Migrate the database, you are advised to stop the instance before doing that +su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate" +``` + +## For from source installations (using git) + 1. Go to the working directory of Pleroma (default is `/opt/pleroma`) 2. Run `git pull`. This pulls the latest changes from upstream. 3. Run `mix deps.get`. This pulls in any new dependencies. diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index 93230806c..aab5197a2 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -259,19 +259,14 @@ su pleroma -s $SHELL -lc "./bin/pleroma_ctl user new joeuser joeuser@sld.tld --a ``` This will create an account withe the username of 'joeuser' with the email address of joeuser@sld.tld, and set that user's account as an admin. This will result in a link that you can paste into the browser, which logs you in and enables you to set the password. -### Updating -Generally, doing the following is enough: -```sh -# Download the new release -su pleroma -s $SHELL -lc "./bin/pleroma_ctl update" - -# Migrate the database, you are advised to stop the instance before doing that -su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate" -``` -But you should **always check the release notes/changelog** in case there are config deprecations, special update steps, etc. - ## Further reading * [Backup your instance](../administration/backup.md) * [Hardening your instance](../configuration/hardening.md) * [How to activate mediaproxy](../configuration/howto_mediaproxy.md) +* [Updating your instance](../administration/updating.md) + +## Questions + +Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**. + From 7bfb8ed14d2d222e6fcd7afe54f229f68370f061 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 21 Jan 2020 16:16:59 +0100 Subject: [PATCH 19/81] remote_follow_controller.ex: Redirect to the user page on success Closes: https://git.pleroma.social/pleroma/pleroma/issues/1245 --- .../controllers/remote_follow_controller.ex | 6 +++--- .../remote_follow_controller_test.exs | 18 +++++++----------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex index e0d4d5632..fbf31c7eb 100644 --- a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do @@ -69,7 +69,7 @@ defp is_status?(acct) do def do_follow(%{assigns: %{user: %User{} = user}} = conn, %{"user" => %{"id" => id}}) do with {:fetch_user, %User{} = followee} <- {:fetch_user, User.get_cached_by_id(id)}, {:ok, _, _, _} <- CommonAPI.follow(user, followee) do - render(conn, "followed.html", %{error: false}) + redirect(conn, to: "/users/#{followee.id}") else error -> handle_follow_error(conn, error) @@ -80,7 +80,7 @@ def do_follow(conn, %{"authorization" => %{"name" => _, "password" => _, "id" => with {:fetch_user, %User{} = followee} <- {:fetch_user, User.get_cached_by_id(id)}, {_, {:ok, user}, _} <- {:auth, Authenticator.get_user(conn), followee}, {:ok, _, _, _} <- CommonAPI.follow(user, followee) do - render(conn, "followed.html", %{error: false}) + redirect(conn, to: "/users/#{followee.id}") else error -> handle_follow_error(conn, error) diff --git a/test/web/twitter_api/remote_follow_controller_test.exs b/test/web/twitter_api/remote_follow_controller_test.exs index 444949375..80a42989d 100644 --- a/test/web/twitter_api/remote_follow_controller_test.exs +++ b/test/web/twitter_api/remote_follow_controller_test.exs @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors +# Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do @@ -92,15 +92,13 @@ test "follows user", %{conn: conn} do user = insert(:user) user2 = insert(:user) - response = + conn = conn |> assign(:user, user) |> assign(:token, insert(:oauth_token, user: user, scopes: ["write:follows"])) |> post(remote_follow_path(conn, :do_follow), %{"user" => %{"id" => user2.id}}) - |> response(200) - assert response =~ "Account followed!" - assert user2.follower_address in User.following(user) + assert redirected_to(conn) == "/users/#{user2.id}" end test "returns error when user is deactivated", %{conn: conn} do @@ -149,14 +147,13 @@ test "returns success result when user already in followers", %{conn: conn} do user2 = insert(:user) {:ok, _, _, _} = CommonAPI.follow(user, user2) - response = + conn = conn |> assign(:user, refresh_record(user)) |> assign(:token, insert(:oauth_token, user: user, scopes: ["write:follows"])) |> post(remote_follow_path(conn, :do_follow), %{"user" => %{"id" => user2.id}}) - |> response(200) - assert response =~ "Account followed!" + assert redirected_to(conn) == "/users/#{user2.id}" end end @@ -165,14 +162,13 @@ test "follows", %{conn: conn} do user = insert(:user) user2 = insert(:user) - response = + conn = conn |> post(remote_follow_path(conn, :do_follow), %{ "authorization" => %{"name" => user.nickname, "password" => "test", "id" => user2.id} }) - |> response(200) - assert response =~ "Account followed!" + assert redirected_to(conn) == "/users/#{user2.id}" assert user2.follower_address in User.following(user) end From df0b8f1d0802a2536bf436ff8157918929a183cc Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 22 Jan 2020 20:31:42 +0400 Subject: [PATCH 20/81] Add new users digest email --- config/config.exs | 6 +- lib/pleroma/emails/new_users_digest_email.ex | 36 ++++ .../templates/email/new_users_digest.html.eex | 158 ++++++++++++++ .../templates/layout/email_styled.html.eex | 193 ++++++++++++++++++ lib/pleroma/web/views/email_view.ex | 4 + .../workers/new_users_digest_worker.ex | 56 +++++ .../cron/new_users_digest_worker_test.exs | 32 +++ 7 files changed, 484 insertions(+), 1 deletion(-) create mode 100644 lib/pleroma/emails/new_users_digest_email.ex create mode 100644 lib/pleroma/web/templates/email/new_users_digest.html.eex create mode 100644 lib/pleroma/web/templates/layout/email_styled.html.eex create mode 100644 lib/pleroma/workers/new_users_digest_worker.ex create mode 100644 test/workers/cron/new_users_digest_worker_test.exs diff --git a/config/config.exs b/config/config.exs index b0036fff0..53ea76dd3 100644 --- a/config/config.exs +++ b/config/config.exs @@ -502,7 +502,11 @@ mailer: 10, transmogrifier: 20, scheduled_activities: 10, - background: 5 + background: 5, + new_users_digest: 1 + ], + crontab: [ + {"0 0 * * *", Pleroma.Workers.NewUsersDigestWorker} ] config :pleroma, :workers, diff --git a/lib/pleroma/emails/new_users_digest_email.ex b/lib/pleroma/emails/new_users_digest_email.ex new file mode 100644 index 000000000..21096a744 --- /dev/null +++ b/lib/pleroma/emails/new_users_digest_email.ex @@ -0,0 +1,36 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Emails.NewUsersDigestEmail do + use Phoenix.Swoosh, view: Pleroma.Web.EmailView, layout: {Pleroma.Web.LayoutView, :email_styled} + + defp instance_notify_email do + Pleroma.Config.get([:instance, :notify_email]) || Pleroma.Config.get([:instance, :email]) + end + + def new_users(to, users_and_statuses) do + instance_name = Pleroma.Config.get([:instance, :name]) + styling = Pleroma.Config.get([Pleroma.Emails.UserEmail, :styling]) + logo = Pleroma.Config.get([Pleroma.Emails.UserEmail, :logo]) + + logo_path = + if is_nil(logo) do + Path.join(:code.priv_dir(:pleroma), "static/static/logo.png") + else + Path.join(Pleroma.Config.get([:instance, :static_dir]), logo) + end + + new() + |> to({to.name, to.email}) + |> from({instance_name, instance_notify_email()}) + |> subject("#{instance_name} New Users") + |> render_body("new_users_digest.html", %{ + title: "New Users", + users_and_statuses: users_and_statuses, + instance: instance_name, + styling: styling + }) + |> attachment(Swoosh.Attachment.new(logo_path, filename: "logo.png", type: :inline)) + end +end diff --git a/lib/pleroma/web/templates/email/new_users_digest.html.eex b/lib/pleroma/web/templates/email/new_users_digest.html.eex new file mode 100644 index 000000000..40d9b8381 --- /dev/null +++ b/lib/pleroma/web/templates/email/new_users_digest.html.eex @@ -0,0 +1,158 @@ +<%= for {user, total_statuses, latest_status} <- @users_and_statuses do %> + <%# user card START %> +
+
+
+ + +
+
+ +
+ +
+ <%= user.name %> + +
+ +
+ +
+
+ + + +
+
+ +
+ + +
+
+

<%= user.name %>

+

<%= link "@" <> user.nickname, style: "color: #{@styling.link_color};text-decoration: none;", to: admin_user_url(user) %>

+

Total: <%= total_statuses %>

+
+
+ + +
+ +
+
+ + +
+
+
+ <%# user card END %> + + <%= if latest_status do %> +
+
+
+ + +
+
+ +
+ + +
+
+ <%= raw latest_status.object.data["content"] %>
+
+ + +
+
+

<%= format_date latest_status.object.data["published"] %>

+
+
+ + +
+ +
+
+ + +
+
+
+ <% end %> + <%# divider start %> +
+
+
+ + +
+
+ +
+ + + + + + + + + +
+ +
+
+ + +
+
+
+ + <%# divider end %> + <%# user card END %> +<% end %> diff --git a/lib/pleroma/web/templates/layout/email_styled.html.eex b/lib/pleroma/web/templates/layout/email_styled.html.eex new file mode 100644 index 000000000..295d2bba0 --- /dev/null +++ b/lib/pleroma/web/templates/layout/email_styled.html.eex @@ -0,0 +1,193 @@ + + + + + + + + + + + + <%= @email.subject %>< + + + + + + + + + + + + + + + + + + + diff --git a/lib/pleroma/web/views/email_view.ex b/lib/pleroma/web/views/email_view.ex index b506a234b..6b0fbe61e 100644 --- a/lib/pleroma/web/views/email_view.ex +++ b/lib/pleroma/web/views/email_view.ex @@ -12,4 +12,8 @@ def format_date(date) when is_binary(date) do |> Timex.parse!("{ISO:Extended:Z}") |> Timex.format!("{Mshort} {D}, {YYYY} {h24}:{m}") end + + def admin_user_url(%{id: id}) do + Pleroma.Web.Endpoint.url() <> "/pleroma/admin/#/users/" <> id + end end diff --git a/lib/pleroma/workers/new_users_digest_worker.ex b/lib/pleroma/workers/new_users_digest_worker.ex new file mode 100644 index 000000000..24cc6bdf3 --- /dev/null +++ b/lib/pleroma/workers/new_users_digest_worker.ex @@ -0,0 +1,56 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.NewUsersDigestWorker do + alias Pleroma.User + alias Pleroma.Repo + alias Pleroma.Activity + + import Ecto.Query + + use Pleroma.Workers.WorkerHelper, queue: "new_users_digest" + + @impl Oban.Worker + def perform(_args, _job) do + today = NaiveDateTime.utc_now() |> Timex.beginning_of_day() + + a_day_ago = + today + |> Timex.shift(days: -1) + |> Timex.beginning_of_day() + + users_and_statuses = + %{ + local: true, + order_by: :inserted_at + } + |> User.Query.build() + |> where([u], u.inserted_at >= ^a_day_ago and u.inserted_at < ^today) + |> Repo.all() + |> Enum.map(fn user -> + latest_status = + Activity + |> Activity.Queries.by_actor(user.ap_id) + |> Activity.Queries.by_type("Create") + |> Activity.with_preloaded_object() + |> order_by(desc: :inserted_at) + |> limit(1) + |> Repo.one() + + total_statuses = + Activity + |> Activity.Queries.by_actor(user.ap_id) + |> Activity.Queries.by_type("Create") + |> Repo.aggregate(:count, :id) + + {user, total_statuses, latest_status} + end) + + %{is_admin: true} + |> User.Query.build() + |> Repo.all() + |> Enum.map(&Pleroma.Emails.NewUsersDigestEmail.new_users(&1, users_and_statuses)) + |> Enum.each(&Pleroma.Emails.Mailer.deliver/1) + end +end diff --git a/test/workers/cron/new_users_digest_worker_test.exs b/test/workers/cron/new_users_digest_worker_test.exs new file mode 100644 index 000000000..7892a7809 --- /dev/null +++ b/test/workers/cron/new_users_digest_worker_test.exs @@ -0,0 +1,32 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Cron.NewUsersDigestWorkerTest do + use Pleroma.DataCase + import Pleroma.Factory + + alias Pleroma.Workers.NewUsersDigestWorker + alias Pleroma.Tests.ObanHelpers + alias Pleroma.Web.CommonAPI + + test "it sends new users digest emails" do + yesterday = NaiveDateTime.utc_now() |> Timex.shift(days: -1) + admin = insert(:user, %{is_admin: true}) + user = insert(:user, %{inserted_at: yesterday}) + user2 = insert(:user, %{inserted_at: yesterday}) + CommonAPI.post(user, %{"status" => "cofe"}) + + NewUsersDigestWorker.perform(nil, nil) + ObanHelpers.perform_all() + + assert_received {:email, email} + assert email.to == [{admin.name, admin.email}] + assert email.subject == "#{Pleroma.Config.get([:instance, :name])} New Users" + + refute email.html_body =~ admin.nickname + assert email.html_body =~ user.nickname + assert email.html_body =~ user2.nickname + assert email.html_body =~ "cofe" + end +end From aa0f0d4edd4205c2b1d7c4f5a885d57287f6379a Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 22 Jan 2020 20:53:06 +0400 Subject: [PATCH 21/81] Disable NewUsersDigestEmail by default --- config/config.exs | 2 + config/test.exs | 2 + .../workers/new_users_digest_worker.ex | 72 ++++++++++--------- 3 files changed, 41 insertions(+), 35 deletions(-) diff --git a/config/config.exs b/config/config.exs index 53ea76dd3..3f0222f0e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -581,6 +581,8 @@ text_muted_color: "#b9b9ba" } +config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: false + config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, path: "/api/pleroma/app_metrics" config :pleroma, Pleroma.ScheduledActivity, diff --git a/config/test.exs b/config/test.exs index 5c66a36f1..9da0ae484 100644 --- a/config/test.exs +++ b/config/test.exs @@ -97,6 +97,8 @@ config :pleroma, :modules, runtime_dir: "test/fixtures/modules" +config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: true + if File.exists?("./config/test.secret.exs") do import_config "test.secret.exs" else diff --git a/lib/pleroma/workers/new_users_digest_worker.ex b/lib/pleroma/workers/new_users_digest_worker.ex index 24cc6bdf3..b776b2c79 100644 --- a/lib/pleroma/workers/new_users_digest_worker.ex +++ b/lib/pleroma/workers/new_users_digest_worker.ex @@ -13,44 +13,46 @@ defmodule Pleroma.Workers.NewUsersDigestWorker do @impl Oban.Worker def perform(_args, _job) do - today = NaiveDateTime.utc_now() |> Timex.beginning_of_day() + if Pleroma.Config.get([Pleroma.Emails.NewUsersDigestEmail, :enabled]) do + today = NaiveDateTime.utc_now() |> Timex.beginning_of_day() - a_day_ago = - today - |> Timex.shift(days: -1) - |> Timex.beginning_of_day() + a_day_ago = + today + |> Timex.shift(days: -1) + |> Timex.beginning_of_day() - users_and_statuses = - %{ - local: true, - order_by: :inserted_at - } + users_and_statuses = + %{ + local: true, + order_by: :inserted_at + } + |> User.Query.build() + |> where([u], u.inserted_at >= ^a_day_ago and u.inserted_at < ^today) + |> Repo.all() + |> Enum.map(fn user -> + latest_status = + Activity + |> Activity.Queries.by_actor(user.ap_id) + |> Activity.Queries.by_type("Create") + |> Activity.with_preloaded_object() + |> order_by(desc: :inserted_at) + |> limit(1) + |> Repo.one() + + total_statuses = + Activity + |> Activity.Queries.by_actor(user.ap_id) + |> Activity.Queries.by_type("Create") + |> Repo.aggregate(:count, :id) + + {user, total_statuses, latest_status} + end) + + %{is_admin: true} |> User.Query.build() - |> where([u], u.inserted_at >= ^a_day_ago and u.inserted_at < ^today) |> Repo.all() - |> Enum.map(fn user -> - latest_status = - Activity - |> Activity.Queries.by_actor(user.ap_id) - |> Activity.Queries.by_type("Create") - |> Activity.with_preloaded_object() - |> order_by(desc: :inserted_at) - |> limit(1) - |> Repo.one() - - total_statuses = - Activity - |> Activity.Queries.by_actor(user.ap_id) - |> Activity.Queries.by_type("Create") - |> Repo.aggregate(:count, :id) - - {user, total_statuses, latest_status} - end) - - %{is_admin: true} - |> User.Query.build() - |> Repo.all() - |> Enum.map(&Pleroma.Emails.NewUsersDigestEmail.new_users(&1, users_and_statuses)) - |> Enum.each(&Pleroma.Emails.Mailer.deliver/1) + |> Enum.map(&Pleroma.Emails.NewUsersDigestEmail.new_users(&1, users_and_statuses)) + |> Enum.each(&Pleroma.Emails.Mailer.deliver/1) + end end end From d43c9ab286c410abfe903b56911888d75f492155 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 22 Jan 2020 16:58:43 +0000 Subject: [PATCH 22/81] Apply suggestion to docs/configuration/cheatsheet.md --- docs/configuration/cheatsheet.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index f690259bf..22f099581 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -515,7 +515,7 @@ Pleroma has the following queues: #### Oban.Cron -Pleroma has the periodic jobs: +Pleroma has these periodic job workers: `Pleroma.Workers.Cron.ClearOauthTokenWorker` - the job to clean an expired oauth tokens. From e761d76e4fe95a8fadd8eb05d5c55ef5fc4bc8ec Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 22 Jan 2020 16:58:50 +0000 Subject: [PATCH 23/81] Apply suggestion to docs/configuration/cheatsheet.md --- docs/configuration/cheatsheet.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 22f099581..48d980aea 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -517,7 +517,7 @@ Pleroma has the following queues: Pleroma has these periodic job workers: -`Pleroma.Workers.Cron.ClearOauthTokenWorker` - the job to clean an expired oauth tokens. +`Pleroma.Workers.Cron.ClearOauthTokenWorker` - a job worker to cleanup expired oauth tokens. Example: From b6f5b326e7b7b7e209a436190d28ac2a165cb057 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 22 Jan 2020 20:59:58 +0400 Subject: [PATCH 24/81] Fix credo warnings --- lib/pleroma/workers/new_users_digest_worker.ex | 4 ++-- test/workers/cron/new_users_digest_worker_test.exs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/workers/new_users_digest_worker.ex b/lib/pleroma/workers/new_users_digest_worker.ex index b776b2c79..845eb4bf7 100644 --- a/lib/pleroma/workers/new_users_digest_worker.ex +++ b/lib/pleroma/workers/new_users_digest_worker.ex @@ -3,9 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.NewUsersDigestWorker do - alias Pleroma.User - alias Pleroma.Repo alias Pleroma.Activity + alias Pleroma.Repo + alias Pleroma.User import Ecto.Query diff --git a/test/workers/cron/new_users_digest_worker_test.exs b/test/workers/cron/new_users_digest_worker_test.exs index 7892a7809..f735cb152 100644 --- a/test/workers/cron/new_users_digest_worker_test.exs +++ b/test/workers/cron/new_users_digest_worker_test.exs @@ -6,9 +6,9 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorkerTest do use Pleroma.DataCase import Pleroma.Factory - alias Pleroma.Workers.NewUsersDigestWorker alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.CommonAPI + alias Pleroma.Workers.NewUsersDigestWorker test "it sends new users digest emails" do yesterday = NaiveDateTime.utc_now() |> Timex.shift(days: -1) From 7c0ac54437e07b85d857700296173035fec5e202 Mon Sep 17 00:00:00 2001 From: minibikini Date: Wed, 22 Jan 2020 17:18:55 +0000 Subject: [PATCH 25/81] Apply suggestion to lib/pleroma/web/templates/layout/email_styled.html.eex --- lib/pleroma/web/templates/layout/email_styled.html.eex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/templates/layout/email_styled.html.eex b/lib/pleroma/web/templates/layout/email_styled.html.eex index 295d2bba0..eb5f59244 100644 --- a/lib/pleroma/web/templates/layout/email_styled.html.eex +++ b/lib/pleroma/web/templates/layout/email_styled.html.eex @@ -11,7 +11,7 @@ - <%= @email.subject %>< + <%= @email.subject %>