From 23d279e03ee1f7a1285614754738711359bc4b81 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 1 Aug 2019 17:28:00 +0300 Subject: [PATCH 01/45] [#1149] Replaced RetryQueue with oban-based retries. --- config/config.exs | 17 +- config/test.exs | 4 + docs/config.md | 7 - lib/pleroma/application.ex | 4 +- lib/pleroma/web/activity_pub/publisher.ex | 16 +- lib/pleroma/web/federator/federator.ex | 14 - lib/pleroma/web/federator/publisher.ex | 22 +- lib/pleroma/web/federator/retry_queue.ex | 239 ------------------ lib/pleroma/web/salmon/salmon.ex | 11 +- lib/pleroma/workers/publisher.ex | 14 + mix.exs | 1 + mix.lock | 1 + .../20190730055101_add_oban_jobs_table.exs | 6 + test/user_test.exs | 15 +- test/web/activity_pub/publisher_test.exs | 2 +- test/web/federator_test.exs | 78 +++--- test/web/retry_queue_test.exs | 48 ---- test/web/salmon/salmon_test.exs | 2 +- 18 files changed, 106 insertions(+), 395 deletions(-) delete mode 100644 lib/pleroma/web/federator/retry_queue.ex create mode 100644 lib/pleroma/workers/publisher.ex create mode 100644 priv/repo/migrations/20190730055101_add_oban_jobs_table.exs delete mode 100644 test/web/retry_queue_test.exs diff --git a/config/config.exs b/config/config.exs index 17770640a..1bb325bf5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -440,13 +440,7 @@ config :pleroma, Pleroma.User, "web" ] -config :pleroma, Pleroma.Web.Federator.RetryQueue, - enabled: false, - max_jobs: 20, - initial_timeout: 30, - max_retries: 5 - -config :pleroma_job_queue, :queues, +job_queues = [ federator_incoming: 50, federator_outgoing: 50, web_push: 50, @@ -454,6 +448,15 @@ config :pleroma_job_queue, :queues, transmogrifier: 20, scheduled_activities: 10, background: 5 +] + +config :pleroma_job_queue, :queues, job_queues + +config :pleroma, Oban, + repo: Pleroma.Repo, + verbose: false, + prune: {:maxage, 60 * 60 * 24 * 7}, + queues: job_queues config :pleroma, :fetch_initial_posts, enabled: false, diff --git a/config/test.exs b/config/test.exs index 92dca18bc..23d9bf779 100644 --- a/config/test.exs +++ b/config/test.exs @@ -62,6 +62,10 @@ config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock config :pleroma_job_queue, disabled: true +config :pleroma, Oban, + queues: false, + prune: :disabled + config :pleroma, Pleroma.ScheduledActivity, daily_user_limit: 2, total_user_limit: 3, diff --git a/docs/config.md b/docs/config.md index 02f86dc16..5c18ffdbf 100644 --- a/docs/config.md +++ b/docs/config.md @@ -412,13 +412,6 @@ config :pleroma_job_queue, :queues, This config contains two queues: `federator_incoming` and `federator_outgoing`. Both have the `max_jobs` set to `50`. -## Pleroma.Web.Federator.RetryQueue - -* `enabled`: If set to `true`, failed federation jobs will be retried -* `max_jobs`: The maximum amount of parallel federation jobs running at the same time. -* `initial_timeout`: The initial timeout in seconds -* `max_retries`: The maximum number of times a federation job is retried - ## Pleroma.Web.Metadata * `providers`: a list of metadata providers to enable. Providers available: * Pleroma.Web.Metadata.Providers.OpenGraph diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 035331491..ce7d8c4b2 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -120,8 +120,8 @@ defmodule Pleroma.Application do hackney_pool_children() ++ [ %{ - id: Pleroma.Web.Federator.RetryQueue, - start: {Pleroma.Web.Federator.RetryQueue, :start_link, []} + id: Oban, + start: {Oban, :start_link, [Application.get_env(:pleroma, Oban)]} }, %{ id: Pleroma.Web.OAuth.Token.CleanWorker, diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 46edab0bd..29f3221d1 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -85,6 +85,15 @@ defmodule Pleroma.Web.ActivityPub.Publisher do end end + def publish_one(%{actor_id: actor_id} = params) do + actor = User.get_by_id(actor_id) + + params + |> Map.delete(:actor_id) + |> Map.put(:actor, actor) + |> publish_one() + end + defp should_federate?(inbox, public) do if public do true @@ -160,7 +169,8 @@ defmodule Pleroma.Web.ActivityPub.Publisher do Publishes an activity with BCC to all relevant peers. """ - def publish(actor, %{data: %{"bcc" => bcc}} = activity) when is_list(bcc) and bcc != [] do + def publish(%User{} = actor, %{data: %{"bcc" => bcc}} = activity) + when is_list(bcc) and bcc != [] do public = is_public?(activity) {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) @@ -187,7 +197,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do Pleroma.Web.Federator.Publisher.enqueue_one(__MODULE__, %{ inbox: inbox, json: json, - actor: actor, + actor_id: actor.id, id: activity.data["id"], unreachable_since: unreachable_since }) @@ -222,7 +232,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do %{ inbox: inbox, json: json, - actor: actor, + actor_id: actor.id, id: activity.data["id"], unreachable_since: unreachable_since } diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index f4f9e83e0..97ec9d549 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -10,7 +10,6 @@ defmodule Pleroma.Web.Federator do alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Federator.Publisher - alias Pleroma.Web.Federator.RetryQueue alias Pleroma.Web.OStatus alias Pleroma.Web.Websub @@ -130,19 +129,6 @@ defmodule Pleroma.Web.Federator do end end - def perform( - :publish_single_websub, - %{xml: _xml, topic: _topic, callback: _callback, secret: _secret} = params - ) do - case Websub.publish_one(params) do - {:ok, _} -> - :ok - - {:error, _} -> - RetryQueue.enqueue(params, Websub) - end - end - def perform(type, _) do Logger.debug(fn -> "Unknown task: #{type}" end) {:error, "Don't know what to do with this"} diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex index 70f870244..e8c1bf17f 100644 --- a/lib/pleroma/web/federator/publisher.ex +++ b/lib/pleroma/web/federator/publisher.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Web.Federator.Publisher do alias Pleroma.Activity alias Pleroma.Config alias Pleroma.User - alias Pleroma.Web.Federator.RetryQueue require Logger @@ -30,23 +29,10 @@ defmodule Pleroma.Web.Federator.Publisher do Enqueue publishing a single activity. """ @spec enqueue_one(module(), Map.t()) :: :ok - def enqueue_one(module, %{} = params), - do: PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish_one, module, params]) - - @spec perform(atom(), module(), any()) :: {:ok, any()} | {:error, any()} - def perform(:publish_one, module, params) do - case apply(module, :publish_one, [params]) do - {:ok, _} -> - :ok - - {:error, _e} -> - RetryQueue.enqueue(params, module) - end - end - - def perform(type, _, _) do - Logger.debug("Unknown task: #{type}") - {:error, "Don't know what to do with this"} + def enqueue_one(module, %{} = params) do + %{module: to_string(module), params: params} + |> Pleroma.Workers.Publisher.new() + |> Pleroma.Repo.insert() end @doc """ diff --git a/lib/pleroma/web/federator/retry_queue.ex b/lib/pleroma/web/federator/retry_queue.ex deleted file mode 100644 index 3db948c2e..000000000 --- a/lib/pleroma/web/federator/retry_queue.ex +++ /dev/null @@ -1,239 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Federator.RetryQueue do - use GenServer - - require Logger - - def init(args) do - queue_table = :ets.new(:pleroma_retry_queue, [:bag, :protected]) - - {:ok, %{args | queue_table: queue_table, running_jobs: :sets.new()}} - end - - def start_link do - enabled = - if Pleroma.Config.get(:env) == :test, - do: true, - else: Pleroma.Config.get([__MODULE__, :enabled], false) - - if enabled do - Logger.info("Starting retry queue") - - linkres = - GenServer.start_link( - __MODULE__, - %{delivered: 0, dropped: 0, queue_table: nil, running_jobs: nil}, - name: __MODULE__ - ) - - maybe_kickoff_timer() - linkres - else - Logger.info("Retry queue disabled") - :ignore - end - end - - def enqueue(data, transport, retries \\ 0) do - GenServer.cast(__MODULE__, {:maybe_enqueue, data, transport, retries + 1}) - end - - def get_stats do - GenServer.call(__MODULE__, :get_stats) - end - - def reset_stats do - GenServer.call(__MODULE__, :reset_stats) - end - - def get_retry_params(retries) do - if retries > Pleroma.Config.get([__MODULE__, :max_retries]) do - {:drop, "Max retries reached"} - else - {:retry, growth_function(retries)} - end - end - - def get_retry_timer_interval do - Pleroma.Config.get([:retry_queue, :interval], 1000) - end - - defp ets_count_expires(table, current_time) do - :ets.select_count( - table, - [ - { - {:"$1", :"$2"}, - [{:"=<", :"$1", {:const, current_time}}], - [true] - } - ] - ) - end - - defp ets_pop_n_expired(table, current_time, desired) do - {popped, _continuation} = - :ets.select( - table, - [ - { - {:"$1", :"$2"}, - [{:"=<", :"$1", {:const, current_time}}], - [:"$_"] - } - ], - desired - ) - - popped - |> Enum.each(fn e -> - :ets.delete_object(table, e) - end) - - popped - end - - def maybe_start_job(running_jobs, queue_table) do - # we don't want to hit the ets or the DateTime more times than we have to - # could optimize slightly further by not using the count, and instead grabbing - # up to N objects early... - current_time = DateTime.to_unix(DateTime.utc_now()) - n_running_jobs = :sets.size(running_jobs) - - if n_running_jobs < Pleroma.Config.get([__MODULE__, :max_jobs]) do - n_ready_jobs = ets_count_expires(queue_table, current_time) - - if n_ready_jobs > 0 do - # figure out how many we could start - available_job_slots = Pleroma.Config.get([__MODULE__, :max_jobs]) - n_running_jobs - start_n_jobs(running_jobs, queue_table, current_time, available_job_slots) - else - running_jobs - end - else - running_jobs - end - end - - defp start_n_jobs(running_jobs, _queue_table, _current_time, 0) do - running_jobs - end - - defp start_n_jobs(running_jobs, queue_table, current_time, available_job_slots) - when available_job_slots > 0 do - candidates = ets_pop_n_expired(queue_table, current_time, available_job_slots) - - candidates - |> List.foldl(running_jobs, fn {_, e}, rj -> - {:ok, pid} = Task.start(fn -> worker(e) end) - mref = Process.monitor(pid) - :sets.add_element(mref, rj) - end) - end - - def worker({:send, data, transport, retries}) do - case transport.publish_one(data) do - {:ok, _} -> - GenServer.cast(__MODULE__, :inc_delivered) - :delivered - - {:error, _reason} -> - enqueue(data, transport, retries) - :retry - end - end - - def handle_call(:get_stats, _from, %{delivered: delivery_count, dropped: drop_count} = state) do - {:reply, %{delivered: delivery_count, dropped: drop_count}, state} - end - - def handle_call(:reset_stats, _from, %{delivered: delivery_count, dropped: drop_count} = state) do - {:reply, %{delivered: delivery_count, dropped: drop_count}, - %{state | delivered: 0, dropped: 0}} - end - - def handle_cast(:reset_stats, state) do - {:noreply, %{state | delivered: 0, dropped: 0}} - end - - def handle_cast( - {:maybe_enqueue, data, transport, retries}, - %{dropped: drop_count, queue_table: queue_table, running_jobs: running_jobs} = state - ) do - case get_retry_params(retries) do - {:retry, timeout} -> - :ets.insert(queue_table, {timeout, {:send, data, transport, retries}}) - running_jobs = maybe_start_job(running_jobs, queue_table) - {:noreply, %{state | running_jobs: running_jobs}} - - {:drop, message} -> - Logger.debug(message) - {:noreply, %{state | dropped: drop_count + 1}} - end - end - - def handle_cast(:kickoff_timer, state) do - retry_interval = get_retry_timer_interval() - Process.send_after(__MODULE__, :retry_timer_run, retry_interval) - {:noreply, state} - end - - def handle_cast(:inc_delivered, %{delivered: delivery_count} = state) do - {:noreply, %{state | delivered: delivery_count + 1}} - end - - def handle_cast(:inc_dropped, %{dropped: drop_count} = state) do - {:noreply, %{state | dropped: drop_count + 1}} - end - - def handle_info({:send, data, transport, retries}, %{delivered: delivery_count} = state) do - case transport.publish_one(data) do - {:ok, _} -> - {:noreply, %{state | delivered: delivery_count + 1}} - - {:error, _reason} -> - enqueue(data, transport, retries) - {:noreply, state} - end - end - - def handle_info( - :retry_timer_run, - %{queue_table: queue_table, running_jobs: running_jobs} = state - ) do - maybe_kickoff_timer() - running_jobs = maybe_start_job(running_jobs, queue_table) - {:noreply, %{state | running_jobs: running_jobs}} - end - - def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do - %{running_jobs: running_jobs, queue_table: queue_table} = state - running_jobs = :sets.del_element(ref, running_jobs) - running_jobs = maybe_start_job(running_jobs, queue_table) - {:noreply, %{state | running_jobs: running_jobs}} - end - - def handle_info(unknown, state) do - Logger.debug("RetryQueue: don't know what to do with #{inspect(unknown)}, ignoring") - {:noreply, state} - end - - if Pleroma.Config.get(:env) == :test do - defp growth_function(_retries) do - _shutit = Pleroma.Config.get([__MODULE__, :initial_timeout]) - DateTime.to_unix(DateTime.utc_now()) - 1 - end - else - defp growth_function(retries) do - round(Pleroma.Config.get([__MODULE__, :initial_timeout]) * :math.pow(retries, 3)) + - DateTime.to_unix(DateTime.utc_now()) - end - end - - defp maybe_kickoff_timer do - GenServer.cast(__MODULE__, :kickoff_timer) - end -end diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index 9b01ebcc6..bbaa293fd 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -170,6 +170,15 @@ defmodule Pleroma.Web.Salmon do end end + def publish_one(%{recipient_id: recipient_id} = params) do + recipient = User.get_by_id(recipient_id) + + params + |> Map.delete(:recipient_id) + |> Map.put(:recipient, recipient) + |> publish_one() + end + def publish_one(_), do: :noop @supported_activities [ @@ -218,7 +227,7 @@ defmodule Pleroma.Web.Salmon do Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end) Publisher.enqueue_one(__MODULE__, %{ - recipient: remote_user, + recipient_id: remote_user.id, feed: feed, unreachable_since: reachable_urls_metadata[remote_user.info.salmon] }) diff --git a/lib/pleroma/workers/publisher.ex b/lib/pleroma/workers/publisher.ex new file mode 100644 index 000000000..639794830 --- /dev/null +++ b/lib/pleroma/workers/publisher.ex @@ -0,0 +1,14 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Publisher do + use Oban.Worker, queue: "federator_outgoing", max_attempts: 5 + + @impl Oban.Worker + def perform(%Oban.Job{args: %{module: module_name, params: params}}) do + module_name + |> String.to_atom() + |> apply(:publish_one, [params]) + end +end diff --git a/mix.exs b/mix.exs index 2a8fe2e9d..1ca7a4a77 100644 --- a/mix.exs +++ b/mix.exs @@ -101,6 +101,7 @@ defmodule Pleroma.Mixfile do {:phoenix_ecto, "~> 4.0"}, {:ecto_sql, "~> 3.1"}, {:postgrex, ">= 0.13.5"}, + {:oban, "~> 0.6"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, {:pbkdf2_elixir, "~> 0.12.3"}, diff --git a/mix.lock b/mix.lock index 65da7be8b..8c0b9734e 100644 --- a/mix.lock +++ b/mix.lock @@ -55,6 +55,7 @@ "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm"}, "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm"}, "nimble_parsec": {:hex, :nimble_parsec, "0.5.0", "90e2eca3d0266e5c53f8fbe0079694740b9c91b6747f2b7e3c5d21966bba8300", [:mix], [], "hexpm"}, + "oban": {:hex, :oban, "0.6.0", "8b9b861355610e703e58a878bc29959f3f0e1b4cd1e90d785cf2bb2498d3b893", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.3", "6706a148809a29c306062862c803406e88f048277f6e85b68faf73291e820b84", [:mix], [], "hexpm"}, "phoenix": {:hex, :phoenix, "1.4.9", "746d098e10741c334d88143d3c94cab1756435f94387a63441792e66ec0ee974", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/priv/repo/migrations/20190730055101_add_oban_jobs_table.exs b/priv/repo/migrations/20190730055101_add_oban_jobs_table.exs new file mode 100644 index 000000000..2f201bd05 --- /dev/null +++ b/priv/repo/migrations/20190730055101_add_oban_jobs_table.exs @@ -0,0 +1,6 @@ +defmodule Pleroma.Repo.Migrations.AddObanJobsTable do + use Ecto.Migration + + defdelegate up, to: Oban.Migrations + defdelegate down, to: Oban.Migrations +end diff --git a/test/user_test.exs b/test/user_test.exs index 556df45fd..70c376384 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -12,9 +12,9 @@ defmodule Pleroma.UserTest do alias Pleroma.Web.CommonAPI use Pleroma.DataCase + use Oban.Testing, repo: Pleroma.Repo import Pleroma.Factory - import Mock setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -1034,11 +1034,7 @@ defmodule Pleroma.UserTest do refute Activity.get_by_id(repeat.id) end - test_with_mock "it sends out User Delete activity", - %{user: user}, - Pleroma.Web.ActivityPub.Publisher, - [:passthrough], - [] do + test "it sends out User Delete activity", %{user: user} do config_path = [:instance, :federating] initial_setting = Pleroma.Config.get(config_path) Pleroma.Config.put(config_path, true) @@ -1048,11 +1044,8 @@ defmodule Pleroma.UserTest do {:ok, _user} = User.delete(user) - assert called( - Pleroma.Web.ActivityPub.Publisher.publish_one(%{ - inbox: "http://mastodon.example.org/inbox" - }) - ) + assert [%{args: %{"params" => %{"inbox" => "http://mastodon.example.org/inbox"}}}] = + all_enqueued(worker: Pleroma.Workers.Publisher) Pleroma.Config.put(config_path, initial_setting) end diff --git a/test/web/activity_pub/publisher_test.exs b/test/web/activity_pub/publisher_test.exs index 36a39c84c..26d019878 100644 --- a/test/web/activity_pub/publisher_test.exs +++ b/test/web/activity_pub/publisher_test.exs @@ -257,7 +257,7 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do assert called( Pleroma.Web.Federator.Publisher.enqueue_one(Publisher, %{ inbox: "https://domain.com/users/nick1/inbox", - actor: actor, + actor_id: actor.id, id: note_activity.data["id"] }) ) diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index 6e143eee4..5c1704548 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -6,7 +6,10 @@ defmodule Pleroma.Web.FederatorTest do alias Pleroma.Instances alias Pleroma.Web.CommonAPI alias Pleroma.Web.Federator + use Pleroma.DataCase + use Oban.Testing, repo: Pleroma.Repo + import Pleroma.Factory import Mock @@ -22,15 +25,6 @@ defmodule Pleroma.Web.FederatorTest do :ok end - describe "Publisher.perform" do - test "call `perform` with unknown task" do - assert { - :error, - "Don't know what to do with this" - } = Pleroma.Web.Federator.Publisher.perform("test", :ok, :ok) - end - end - describe "Publish an activity" do setup do user = insert(:user) @@ -73,10 +67,7 @@ defmodule Pleroma.Web.FederatorTest do end describe "Targets reachability filtering in `publish`" do - test_with_mock "it federates only to reachable instances via AP", - Pleroma.Web.ActivityPub.Publisher, - [:passthrough], - [] do + test "it federates only to reachable instances via AP" do user = insert(:user) {inbox1, inbox2} = @@ -104,20 +95,13 @@ defmodule Pleroma.Web.FederatorTest do {:ok, _activity} = CommonAPI.post(user, %{"status" => "HI @nick1@domain.com, @nick2@domain2.com!"}) - assert called( - Pleroma.Web.ActivityPub.Publisher.publish_one(%{ - inbox: inbox1, - unreachable_since: dt - }) - ) + expected_dt = NaiveDateTime.to_iso8601(dt) - refute called(Pleroma.Web.ActivityPub.Publisher.publish_one(%{inbox: inbox2})) + assert [%{args: %{"params" => %{"inbox" => ^inbox1, "unreachable_since" => ^expected_dt}}}] = + all_enqueued(worker: Pleroma.Workers.Publisher) end - test_with_mock "it federates only to reachable instances via Websub", - Pleroma.Web.Websub, - [:passthrough], - [] do + test "it federates only to reachable instances via Websub" do user = insert(:user) websub_topic = Pleroma.Web.OStatus.feed_path(user) @@ -142,23 +126,25 @@ defmodule Pleroma.Web.FederatorTest do {:ok, _activity} = CommonAPI.post(user, %{"status" => "HI"}) - assert called( - Pleroma.Web.Websub.publish_one(%{ - callback: sub2.callback, - unreachable_since: dt - }) - ) + expected_callback = sub2.callback + expected_dt = NaiveDateTime.to_iso8601(dt) - refute called(Pleroma.Web.Websub.publish_one(%{callback: sub1.callback})) + assert [ + %{ + args: %{ + "params" => %{ + "callback" => ^expected_callback, + "unreachable_since" => ^expected_dt + } + } + } + ] = all_enqueued(worker: Pleroma.Workers.Publisher) end - test_with_mock "it federates only to reachable instances via Salmon", - Pleroma.Web.Salmon, - [:passthrough], - [] do + test "it federates only to reachable instances via Salmon" do user = insert(:user) - remote_user1 = + _remote_user1 = insert(:user, %{ local: false, nickname: "nick1@domain.com", @@ -174,6 +160,8 @@ defmodule Pleroma.Web.FederatorTest do info: %{salmon: "https://domain2.com/salmon"} }) + remote_user2_id = remote_user2.id + dt = NaiveDateTime.utc_now() Instances.set_unreachable(remote_user2.ap_id, dt) @@ -182,14 +170,18 @@ defmodule Pleroma.Web.FederatorTest do {:ok, _activity} = CommonAPI.post(user, %{"status" => "HI @nick1@domain.com, @nick2@domain2.com!"}) - assert called( - Pleroma.Web.Salmon.publish_one(%{ - recipient: remote_user2, - unreachable_since: dt - }) - ) + expected_dt = NaiveDateTime.to_iso8601(dt) - refute called(Pleroma.Web.Salmon.publish_one(%{recipient: remote_user1})) + assert [ + %{ + args: %{ + "params" => %{ + "recipient_id" => ^remote_user2_id, + "unreachable_since" => ^expected_dt + } + } + } + ] = all_enqueued(worker: Pleroma.Workers.Publisher) end end diff --git a/test/web/retry_queue_test.exs b/test/web/retry_queue_test.exs deleted file mode 100644 index ecb3ce5d0..000000000 --- a/test/web/retry_queue_test.exs +++ /dev/null @@ -1,48 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule MockActivityPub do - def publish_one({ret, waiter}) do - send(waiter, :complete) - {ret, "success"} - end -end - -defmodule Pleroma.Web.Federator.RetryQueueTest do - use Pleroma.DataCase - alias Pleroma.Web.Federator.RetryQueue - - @small_retry_count 0 - @hopeless_retry_count 10 - - setup do - RetryQueue.reset_stats() - end - - test "RetryQueue responds to stats request" do - assert %{delivered: 0, dropped: 0} == RetryQueue.get_stats() - end - - test "failed posts are retried" do - {:retry, _timeout} = RetryQueue.get_retry_params(@small_retry_count) - - wait_task = - Task.async(fn -> - receive do - :complete -> :ok - end - end) - - RetryQueue.enqueue({:ok, wait_task.pid}, MockActivityPub, @small_retry_count) - Task.await(wait_task) - assert %{delivered: 1, dropped: 0} == RetryQueue.get_stats() - end - - test "posts that have been tried too many times are dropped" do - {:drop, _timeout} = RetryQueue.get_retry_params(@hopeless_retry_count) - - RetryQueue.enqueue({:ok, nil}, MockActivityPub, @hopeless_retry_count) - assert %{delivered: 0, dropped: 1} == RetryQueue.get_stats() - end -end diff --git a/test/web/salmon/salmon_test.exs b/test/web/salmon/salmon_test.exs index e86e76fe9..0186f3fef 100644 --- a/test/web/salmon/salmon_test.exs +++ b/test/web/salmon/salmon_test.exs @@ -96,6 +96,6 @@ defmodule Pleroma.Web.Salmon.SalmonTest do Salmon.publish(user, activity) - assert called(Publisher.enqueue_one(Salmon, %{recipient: mentioned_user})) + assert called(Publisher.enqueue_one(Salmon, %{recipient_id: mentioned_user.id})) end end From b7fad8d395c2bd1afe445a370e539571f5ec0c18 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 9 Aug 2019 20:08:01 +0300 Subject: [PATCH 02/45] [#1149] Oban jobs implementation for :federator_incoming and :federator_outgoing queues. --- config/config.exs | 7 + lib/pleroma/web/activity_pub/utils.ex | 9 +- lib/pleroma/web/federator/federator.ex | 134 +++++------------- lib/pleroma/web/federator/publisher.ex | 12 +- lib/pleroma/workers/publisher.ex | 25 +++- lib/pleroma/workers/receiver.ex | 61 ++++++++ lib/pleroma/workers/subscriber.ex | 44 ++++++ test/activity_test.exs | 4 +- test/support/oban_helpers.ex | 36 +++++ test/user_test.exs | 11 +- .../activity_pub_controller_test.exs | 14 +- test/web/federator_test.exs | 57 +++++--- test/web/websub/websub_test.exs | 4 + 13 files changed, 280 insertions(+), 138 deletions(-) create mode 100644 lib/pleroma/workers/receiver.ex create mode 100644 lib/pleroma/workers/subscriber.ex create mode 100644 test/support/oban_helpers.ex diff --git a/config/config.exs b/config/config.exs index 1bb325bf5..5fd64365c 100644 --- a/config/config.exs +++ b/config/config.exs @@ -458,6 +458,13 @@ config :pleroma, Oban, prune: {:maxage, 60 * 60 * 24 * 7}, queues: job_queues +config :pleroma, :workers, + retries: [ + compile_time_default: 1, + federator_incoming: 5, + federator_outgoing: 5 + ] + config :pleroma, :fetch_initial_posts, enabled: false, pages: 5 diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 39074888b..f0917f9d4 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -168,14 +168,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do """ def maybe_federate(%Activity{local: true} = activity) do if Pleroma.Config.get!([:instance, :federating]) do - priority = - case activity.data["type"] do - "Delete" -> 10 - "Create" -> 1 - _ -> 5 - end - - Pleroma.Web.Federator.publish(activity, priority) + Pleroma.Web.Federator.publish(activity) end :ok diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 97ec9d549..bb9eadfee 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -3,22 +3,15 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Federator do - alias Pleroma.Activity - alias Pleroma.Object.Containment - alias Pleroma.User - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.ActivityPub.Transmogrifier - alias Pleroma.Web.ActivityPub.Utils - alias Pleroma.Web.Federator.Publisher - alias Pleroma.Web.OStatus - alias Pleroma.Web.Websub + alias Pleroma.Workers.Publisher, as: PublisherWorker + alias Pleroma.Workers.Receiver, as: ReceiverWorker + alias Pleroma.Workers.Subscriber, as: SubscriberWorker require Logger def init do # 1 minute - Process.sleep(1000 * 60) - refresh_subscriptions() + refresh_subscriptions(schedule_in: 60) end @doc "Addresses [memory leaks on recursive replies fetching](https://git.pleroma.social/pleroma/pleroma/issues/161)" @@ -36,111 +29,50 @@ defmodule Pleroma.Web.Federator do # Client API def incoming_doc(doc) do - PleromaJobQueue.enqueue(:federator_incoming, __MODULE__, [:incoming_doc, doc]) + %{"op" => "incoming_doc", "body" => doc} + |> ReceiverWorker.new(worker_args(:federator_incoming)) + |> Pleroma.Repo.insert() end def incoming_ap_doc(params) do - PleromaJobQueue.enqueue(:federator_incoming, __MODULE__, [:incoming_ap_doc, params]) + %{"op" => "incoming_ap_doc", "params" => params} + |> ReceiverWorker.new(worker_args(:federator_incoming)) + |> Pleroma.Repo.insert() end - def publish(activity, priority \\ 1) do - PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish, activity], priority) + def publish(%{id: "pleroma:fakeid"} = activity) do + PublisherWorker.perform_publish(activity) + end + + def publish(activity) do + %{"op" => "publish", "activity_id" => activity.id} + |> PublisherWorker.new(worker_args(:federator_outgoing)) + |> Pleroma.Repo.insert() end def verify_websub(websub) do - PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:verify_websub, websub]) + %{"op" => "verify_websub", "websub_id" => websub.id} + |> SubscriberWorker.new(worker_args(:federator_outgoing)) + |> Pleroma.Repo.insert() end - def request_subscription(sub) do - PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:request_subscription, sub]) + def request_subscription(websub) do + %{"op" => "request_subscription", "websub_id" => websub.id} + |> SubscriberWorker.new(worker_args(:federator_outgoing)) + |> Pleroma.Repo.insert() end - def refresh_subscriptions do - PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:refresh_subscriptions]) + def refresh_subscriptions(worker_args \\ []) do + %{"op" => "refresh_subscriptions"} + |> SubscriberWorker.new(worker_args ++ [max_attempts: 1] ++ worker_args(:federator_outgoing)) + |> Pleroma.Repo.insert() end - # Job Worker Callbacks - - def perform(:refresh_subscriptions) do - Logger.debug("Federator running refresh subscriptions") - Websub.refresh_subscriptions() - - spawn(fn -> - # 6 hours - Process.sleep(1000 * 60 * 60 * 6) - refresh_subscriptions() - end) - end - - def perform(:request_subscription, websub) do - Logger.debug("Refreshing #{websub.topic}") - - with {:ok, websub} <- Websub.request_subscription(websub) do - Logger.debug("Successfully refreshed #{websub.topic}") + defp worker_args(queue) do + if max_attempts = Pleroma.Config.get([:workers, :retries, queue]) do + [max_attempts: max_attempts] else - _e -> Logger.debug("Couldn't refresh #{websub.topic}") - end - end - - def perform(:publish, activity) do - Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end) - - with %User{} = actor <- User.get_cached_by_ap_id(activity.data["actor"]), - {:ok, actor} <- User.ensure_keys_present(actor) do - Publisher.publish(actor, activity) - end - end - - def perform(:verify_websub, websub) do - Logger.debug(fn -> - "Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})" - end) - - Websub.verify(websub) - end - - def perform(:incoming_doc, doc) do - Logger.info("Got document, trying to parse") - OStatus.handle_incoming(doc) - end - - def perform(:incoming_ap_doc, params) do - Logger.info("Handling incoming AP activity") - - params = Utils.normalize_params(params) - - # NOTE: we use the actor ID to do the containment, this is fine because an - # actor shouldn't be acting on objects outside their own AP server. - with {:ok, _user} <- ap_enabled_actor(params["actor"]), - nil <- Activity.normalize(params["id"]), - :ok <- Containment.contain_origin_from_id(params["actor"], params), - {:ok, activity} <- Transmogrifier.handle_incoming(params) do - {:ok, activity} - else - %Activity{} -> - Logger.info("Already had #{params["id"]}") - :error - - _e -> - # Just drop those for now - Logger.info("Unhandled activity") - Logger.info(Jason.encode!(params, pretty: true)) - :error - end - end - - def perform(type, _) do - Logger.debug(fn -> "Unknown task: #{type}" end) - {:error, "Don't know what to do with this"} - end - - def ap_enabled_actor(id) do - user = User.get_cached_by_ap_id(id) - - if User.ap_enabled?(user) do - {:ok, user} - else - ActivityPub.make_user_from_ap_id(id) + [] end end end diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex index e8c1bf17f..05d2be615 100644 --- a/lib/pleroma/web/federator/publisher.ex +++ b/lib/pleroma/web/federator/publisher.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.Federator.Publisher do alias Pleroma.Activity alias Pleroma.Config alias Pleroma.User + alias Pleroma.Workers.Publisher, as: PublisherWorker require Logger @@ -30,8 +31,15 @@ defmodule Pleroma.Web.Federator.Publisher do """ @spec enqueue_one(module(), Map.t()) :: :ok def enqueue_one(module, %{} = params) do - %{module: to_string(module), params: params} - |> Pleroma.Workers.Publisher.new() + worker_args = + if max_attempts = Pleroma.Config.get([:workers, :retries, :federator_outgoing]) do + [max_attempts: max_attempts] + else + [] + end + + %{"op" => "publish_one", "module" => to_string(module), "params" => params} + |> PublisherWorker.new(worker_args) |> Pleroma.Repo.insert() end diff --git a/lib/pleroma/workers/publisher.ex b/lib/pleroma/workers/publisher.ex index 639794830..67871977a 100644 --- a/lib/pleroma/workers/publisher.ex +++ b/lib/pleroma/workers/publisher.ex @@ -3,12 +3,33 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.Publisher do - use Oban.Worker, queue: "federator_outgoing", max_attempts: 5 + alias Pleroma.Activity + alias Pleroma.User + + # Note: `max_attempts` is intended to be overridden in `new/1` call + use Oban.Worker, + queue: "federator_outgoing", + max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) @impl Oban.Worker - def perform(%Oban.Job{args: %{module: module_name, params: params}}) do + def perform(%{"op" => "publish", "activity_id" => activity_id}) do + with %Activity{} = activity <- Activity.get_by_id(activity_id) do + perform_publish(activity) + else + _ -> raise "Non-existing activity: #{activity_id}" + end + end + + def perform(%{"op" => "publish_one", "module" => module_name, "params" => params}) do module_name |> String.to_atom() |> apply(:publish_one, [params]) end + + def perform_publish(%Activity{} = activity) do + with %User{} = actor <- User.get_cached_by_ap_id(activity.data["actor"]), + {:ok, actor} <- User.ensure_keys_present(actor) do + Pleroma.Web.Federator.Publisher.publish(actor, activity) + end + end end diff --git a/lib/pleroma/workers/receiver.ex b/lib/pleroma/workers/receiver.ex new file mode 100644 index 000000000..43558b4e6 --- /dev/null +++ b/lib/pleroma/workers/receiver.ex @@ -0,0 +1,61 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Receiver do + alias Pleroma.Activity + alias Pleroma.Object.Containment + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Web.OStatus + + require Logger + + # Note: `max_attempts` is intended to be overridden in `new/1` call + use Oban.Worker, + queue: "federator_incoming", + max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + + @impl Oban.Worker + def perform(%{"op" => "incoming_doc", "body" => doc}) do + Logger.info("Got incoming document, trying to parse") + OStatus.handle_incoming(doc) + end + + def perform(%{"op" => "incoming_ap_doc", "params" => params}) do + Logger.info("Handling incoming AP activity") + + params = Utils.normalize_params(params) + + # NOTE: we use the actor ID to do the containment, this is fine because an + # actor shouldn't be acting on objects outside their own AP server. + with {:ok, _user} <- ap_enabled_actor(params["actor"]), + nil <- Activity.normalize(params["id"]), + :ok <- Containment.contain_origin_from_id(params["actor"], params), + {:ok, activity} <- Transmogrifier.handle_incoming(params) do + {:ok, activity} + else + %Activity{} -> + Logger.info("Already had #{params["id"]}") + :error + + _e -> + # Just drop those for now + Logger.info("Unhandled activity") + Logger.info(Jason.encode!(params, pretty: true)) + :error + end + end + + defp ap_enabled_actor(id) do + user = User.get_cached_by_ap_id(id) + + if User.ap_enabled?(user) do + {:ok, user} + else + ActivityPub.make_user_from_ap_id(id) + end + end +end diff --git a/lib/pleroma/workers/subscriber.ex b/lib/pleroma/workers/subscriber.ex new file mode 100644 index 000000000..a8c01bb10 --- /dev/null +++ b/lib/pleroma/workers/subscriber.ex @@ -0,0 +1,44 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Subscriber do + alias Pleroma.Repo + alias Pleroma.Web.Websub + alias Pleroma.Web.Websub.WebsubClientSubscription + + require Logger + + # Note: `max_attempts` is intended to be overridden in `new/1` call + use Oban.Worker, + queue: "federator_outgoing", + max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + + @impl Oban.Worker + def perform(%{"op" => "refresh_subscriptions"}) do + Websub.refresh_subscriptions() + # Schedule the next run in 6 hours + Pleroma.Web.Federator.refresh_subscriptions(schedule_in: 3600 * 6) + end + + def perform(%{"op" => "request_subscription", "websub_id" => websub_id}) do + websub = Repo.get(WebsubClientSubscription, websub_id) + Logger.debug("Refreshing #{websub.topic}") + + with {:ok, websub} <- Websub.request_subscription(websub) do + Logger.debug("Successfully refreshed #{websub.topic}") + else + _e -> Logger.debug("Couldn't refresh #{websub.topic}") + end + end + + def perform(%{"op" => "verify_websub", "websub_id" => websub_id}) do + websub = Repo.get(WebsubClientSubscription, websub_id) + + Logger.debug(fn -> + "Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})" + end) + + Websub.verify(websub) + end +end diff --git a/test/activity_test.exs b/test/activity_test.exs index b27f6fd36..b9c12adb2 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.ActivityTest do use Pleroma.DataCase alias Pleroma.Activity alias Pleroma.Bookmark + alias Pleroma.ObanHelpers alias Pleroma.Object alias Pleroma.ThreadMute import Pleroma.Factory @@ -125,7 +126,8 @@ defmodule Pleroma.ActivityTest do } {:ok, local_activity} = Pleroma.Web.CommonAPI.post(user, %{"status" => "find me!"}) - {:ok, remote_activity} = Pleroma.Web.Federator.incoming_ap_doc(params) + {:ok, job} = Pleroma.Web.Federator.incoming_ap_doc(params) + {:ok, remote_activity} = ObanHelpers.perform(job) %{local_activity: local_activity, remote_activity: remote_activity, user: user} end diff --git a/test/support/oban_helpers.ex b/test/support/oban_helpers.ex new file mode 100644 index 000000000..54b5a9566 --- /dev/null +++ b/test/support/oban_helpers.ex @@ -0,0 +1,36 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ObanHelpers do + @moduledoc """ + Oban test helpers. + """ + + alias Pleroma.Repo + + def perform(%Oban.Job{} = job) do + res = apply(String.to_existing_atom("Elixir." <> job.worker), :perform, [job]) + Repo.delete(job) + res + end + + def perform(jobs) when is_list(jobs) do + for job <- jobs, do: perform(job) + end + + def member?(%{} = job_args, jobs) when is_list(jobs) do + Enum.any?(jobs, fn job -> + member?(job_args, job.args) + end) + end + + def member?(%{} = test_attrs, %{} = attrs) do + Enum.all?( + test_attrs, + fn {k, _v} -> member?(test_attrs[k], attrs[k]) end + ) + end + + def member?(x, y), do: x == y +end diff --git a/test/user_test.exs b/test/user_test.exs index 70c376384..ee6d8e8f3 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.UserTest do alias Pleroma.Activity alias Pleroma.Builders.UserBuilder + alias Pleroma.ObanHelpers alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User @@ -1044,8 +1045,16 @@ defmodule Pleroma.UserTest do {:ok, _user} = User.delete(user) - assert [%{args: %{"params" => %{"inbox" => "http://mastodon.example.org/inbox"}}}] = + assert ObanHelpers.member?( + %{ + "op" => "publish_one", + "params" => %{ + "inbox" => "http://mastodon.example.org/inbox", + "id" => "pleroma:fakeid" + } + }, all_enqueued(worker: Pleroma.Workers.Publisher) + ) Pleroma.Config.put(config_path, initial_setting) end diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 40344f17e..1d809164f 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -4,15 +4,19 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do use Pleroma.Web.ConnCase + use Oban.Testing, repo: Pleroma.Repo + import Pleroma.Factory alias Pleroma.Activity alias Pleroma.Instances + alias Pleroma.ObanHelpers alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.ActivityPub.ObjectView alias Pleroma.Web.ActivityPub.UserView alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.CommonAPI + alias Pleroma.Workers.Receiver, as: ReceiverWorker setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -232,7 +236,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do |> post("/inbox", data) assert "ok" == json_response(conn, 200) - :timer.sleep(500) + + ObanHelpers.perform(all_enqueued(worker: ReceiverWorker)) assert Activity.get_by_ap_id(data["id"]) end @@ -274,7 +279,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do |> post("/users/#{user.nickname}/inbox", data) assert "ok" == json_response(conn, 200) - :timer.sleep(500) + ObanHelpers.perform(all_enqueued(worker: ReceiverWorker)) assert Activity.get_by_ap_id(data["id"]) end @@ -303,7 +308,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do |> post("/users/#{recipient.nickname}/inbox", data) assert "ok" == json_response(conn, 200) - :timer.sleep(500) + ObanHelpers.perform(all_enqueued(worker: ReceiverWorker)) assert Activity.get_by_ap_id(data["id"]) end @@ -382,6 +387,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do |> post("/users/#{recipient.nickname}/inbox", data) |> json_response(200) + ObanHelpers.perform(all_enqueued(worker: ReceiverWorker)) + activity = Activity.get_by_ap_id(data["id"]) assert activity.id @@ -457,6 +464,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do |> post("/users/#{user.nickname}/outbox", data) result = json_response(conn, 201) + assert Activity.get_by_ap_id(result["id"]) end diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index 5c1704548..ebe962da2 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -4,8 +4,10 @@ defmodule Pleroma.Web.FederatorTest do alias Pleroma.Instances + alias Pleroma.ObanHelpers alias Pleroma.Web.CommonAPI alias Pleroma.Web.Federator + alias Pleroma.Workers.Publisher, as: PublisherWorker use Pleroma.DataCase use Oban.Testing, repo: Pleroma.Repo @@ -45,6 +47,7 @@ defmodule Pleroma.Web.FederatorTest do } do with_mocks([relay_mock]) do Federator.publish(activity) + ObanHelpers.perform(all_enqueued(worker: PublisherWorker)) end assert_received :relay_publish @@ -58,6 +61,7 @@ defmodule Pleroma.Web.FederatorTest do with_mocks([relay_mock]) do Federator.publish(activity) + ObanHelpers.perform(all_enqueued(worker: PublisherWorker)) end refute_received :relay_publish @@ -97,8 +101,15 @@ defmodule Pleroma.Web.FederatorTest do expected_dt = NaiveDateTime.to_iso8601(dt) - assert [%{args: %{"params" => %{"inbox" => ^inbox1, "unreachable_since" => ^expected_dt}}}] = - all_enqueued(worker: Pleroma.Workers.Publisher) + ObanHelpers.perform(all_enqueued(worker: PublisherWorker)) + + assert ObanHelpers.member?( + %{ + "op" => "publish_one", + "params" => %{"inbox" => inbox1, "unreachable_since" => expected_dt} + }, + all_enqueued(worker: PublisherWorker) + ) end test "it federates only to reachable instances via Websub" do @@ -129,16 +140,18 @@ defmodule Pleroma.Web.FederatorTest do expected_callback = sub2.callback expected_dt = NaiveDateTime.to_iso8601(dt) - assert [ + ObanHelpers.perform(all_enqueued(worker: PublisherWorker)) + + assert ObanHelpers.member?( %{ - args: %{ - "params" => %{ - "callback" => ^expected_callback, - "unreachable_since" => ^expected_dt - } + "op" => "publish_one", + "params" => %{ + "callback" => expected_callback, + "unreachable_since" => expected_dt } - } - ] = all_enqueued(worker: Pleroma.Workers.Publisher) + }, + all_enqueued(worker: PublisherWorker) + ) end test "it federates only to reachable instances via Salmon" do @@ -172,16 +185,18 @@ defmodule Pleroma.Web.FederatorTest do expected_dt = NaiveDateTime.to_iso8601(dt) - assert [ + ObanHelpers.perform(all_enqueued(worker: PublisherWorker)) + + assert ObanHelpers.member?( %{ - args: %{ - "params" => %{ - "recipient_id" => ^remote_user2_id, - "unreachable_since" => ^expected_dt - } + "op" => "publish_one", + "params" => %{ + "recipient_id" => remote_user2_id, + "unreachable_since" => expected_dt } - } - ] = all_enqueued(worker: Pleroma.Workers.Publisher) + }, + all_enqueued(worker: PublisherWorker) + ) end end @@ -201,7 +216,8 @@ defmodule Pleroma.Web.FederatorTest do "to" => ["https://www.w3.org/ns/activitystreams#Public"] } - {:ok, _activity} = Federator.incoming_ap_doc(params) + assert {:ok, job} = Federator.incoming_ap_doc(params) + assert {:ok, _activity} = ObanHelpers.perform(job) end test "rejects incoming AP docs with incorrect origin" do @@ -219,7 +235,8 @@ defmodule Pleroma.Web.FederatorTest do "to" => ["https://www.w3.org/ns/activitystreams#Public"] } - :error = Federator.incoming_ap_doc(params) + assert {:ok, job} = Federator.incoming_ap_doc(params) + assert :error = ObanHelpers.perform(job) end end end diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index 74386d7db..b704a558a 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -4,11 +4,14 @@ defmodule Pleroma.Web.WebsubTest do use Pleroma.DataCase + use Oban.Testing, repo: Pleroma.Repo + alias Pleroma.ObanHelpers alias Pleroma.Web.Router.Helpers alias Pleroma.Web.Websub alias Pleroma.Web.Websub.WebsubClientSubscription alias Pleroma.Web.Websub.WebsubServerSubscription + alias Pleroma.Workers.Subscriber, as: SubscriberWorker import Pleroma.Factory import Tesla.Mock @@ -224,6 +227,7 @@ defmodule Pleroma.Web.WebsubTest do }) _refresh = Websub.refresh_subscriptions() + ObanHelpers.perform(all_enqueued(worker: SubscriberWorker)) assert still_good == Repo.get(WebsubClientSubscription, still_good.id) refute needs_refresh == Repo.get(WebsubClientSubscription, needs_refresh.id) From 33a5fc4a70b6f9b8c2d8c03a412d7eec8d5b3db1 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 10 Aug 2019 20:38:31 +0300 Subject: [PATCH 03/45] [#1149] Fixed failing tests. Ensured Instance.set_unreachable/2 supports ISO 8601 datetime. --- lib/pleroma/digest_email_worker.ex | 4 +--- lib/pleroma/instances/instance.ex | 8 +++++++- test/conversation_test.exs | 2 ++ test/support/oban_helpers.ex | 6 ++++++ test/web/federator_test.exs | 3 ++- test/web/instances/instance_test.exs | 3 ++- 6 files changed, 20 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/digest_email_worker.ex b/lib/pleroma/digest_email_worker.ex index 18e67d39b..3b0e2bca6 100644 --- a/lib/pleroma/digest_email_worker.ex +++ b/lib/pleroma/digest_email_worker.ex @@ -1,8 +1,6 @@ defmodule Pleroma.DigestEmailWorker do import Ecto.Query - @queue_name :digest_emails - def perform do config = Pleroma.Config.get([:email_notifications, :digest]) negative_interval = -Map.fetch!(config, :interval) @@ -17,7 +15,7 @@ defmodule Pleroma.DigestEmailWorker do select: u ) |> Pleroma.Repo.all() - |> Enum.each(&PleromaJobQueue.enqueue(@queue_name, __MODULE__, [&1])) + |> Enum.each(&PleromaJobQueue.enqueue(:digest_emails, __MODULE__, [&1])) end @doc """ diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 4d7ed4ca1..544c4b687 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -90,7 +90,7 @@ defmodule Pleroma.Instances.Instance do def set_unreachable(url_or_host, unreachable_since \\ nil) def set_unreachable(url_or_host, unreachable_since) when is_binary(url_or_host) do - unreachable_since = unreachable_since || DateTime.utc_now() + unreachable_since = parse_datetime(unreachable_since) || NaiveDateTime.utc_now() host = host(url_or_host) existing_record = Repo.get_by(Instance, %{host: host}) @@ -114,4 +114,10 @@ defmodule Pleroma.Instances.Instance do end def set_unreachable(_, _), do: {:error, nil} + + defp parse_datetime(datetime) when is_binary(datetime) do + NaiveDateTime.from_iso8601(datetime) + end + + defp parse_datetime(datetime), do: datetime end diff --git a/test/conversation_test.exs b/test/conversation_test.exs index aa193e0d4..2ebbcab76 100644 --- a/test/conversation_test.exs +++ b/test/conversation_test.exs @@ -28,6 +28,8 @@ defmodule Pleroma.ConversationTest do {:ok, _activity} = CommonAPI.post(user, %{"visibility" => "direct", "status" => "hey @#{other_user.nickname}"}) + Pleroma.ObanHelpers.perform_all() + Repo.delete_all(Conversation) Repo.delete_all(Conversation.Participation) diff --git a/test/support/oban_helpers.ex b/test/support/oban_helpers.ex index 54b5a9566..ecc03ba1a 100644 --- a/test/support/oban_helpers.ex +++ b/test/support/oban_helpers.ex @@ -9,6 +9,12 @@ defmodule Pleroma.ObanHelpers do alias Pleroma.Repo + def perform_all do + Oban.Job + |> Repo.all() + |> perform() + end + def perform(%Oban.Job{} = job) do res = apply(String.to_existing_atom("Elixir." <> job.worker), :perform, [job]) Repo.delete(job) diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index d3a28d50e..e0be4342b 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -249,7 +249,8 @@ defmodule Pleroma.Web.FederatorTest do File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() - assert Federator.incoming_ap_doc(params) == :error + assert {:ok, job} = Federator.incoming_ap_doc(params) + assert :error = ObanHelpers.perform(job) Pleroma.Config.put([:instance, :rewrite_policy], policies) Pleroma.Config.put(:mrf_keyword, mrf_keyword_policy) diff --git a/test/web/instances/instance_test.exs b/test/web/instances/instance_test.exs index d28730994..a1bdd45d3 100644 --- a/test/web/instances/instance_test.exs +++ b/test/web/instances/instance_test.exs @@ -22,7 +22,8 @@ defmodule Pleroma.Instances.InstanceTest do describe "set_reachable/1" do test "clears `unreachable_since` of existing matching Instance record having non-nil `unreachable_since`" do - instance = insert(:instance, unreachable_since: NaiveDateTime.utc_now()) + unreachable_since = NaiveDateTime.to_iso8601(NaiveDateTime.utc_now()) + instance = insert(:instance, unreachable_since: unreachable_since) assert {:ok, instance} = Instance.set_reachable(instance.host) refute instance.unreachable_since From 0e1c481a94392b69833fbe6afc184ebbd90e1330 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 13 Aug 2019 20:20:26 +0300 Subject: [PATCH 04/45] [#1149] Added more oban workers. Refactoring. --- lib/pleroma/digest_email_worker.ex | 11 ++- lib/pleroma/scheduled_activity_worker.ex | 8 +- lib/pleroma/user.ex | 55 +++++++---- lib/pleroma/web/activity_pub/activity_pub.ex | 7 +- .../mrf/mediaproxy_warming_policy.ex | 12 ++- .../web/activity_pub/transmogrifier.ex | 7 +- lib/pleroma/web/federator/federator.ex | 98 ++++++++++++++++++- lib/pleroma/web/oauth/token/clean_worker.ex | 10 +- lib/pleroma/web/push/push.ex | 12 ++- .../controllers/util_controller.ex | 14 +-- lib/pleroma/workers/background_worker.ex | 66 +++++++++++++ lib/pleroma/workers/helper.ex | 13 +++ lib/pleroma/workers/mailer.ex | 18 ++++ lib/pleroma/workers/publisher.ex | 20 +--- lib/pleroma/workers/receiver.ex | 46 +-------- .../workers/scheduled_activity_worker.ex | 15 +++ lib/pleroma/workers/subscriber.ex | 23 +---- lib/pleroma/workers/transmogrifier.ex | 18 ++++ lib/pleroma/workers/web_pusher.ex | 19 ++++ test/activity_test.exs | 2 +- test/conversation_test.exs | 2 +- test/notification_test.exs | 5 +- test/support/oban_helpers.ex | 2 +- test/user_test.exs | 19 ++-- .../activity_pub_controller_test.exs | 2 +- .../mrf/mediaproxy_warming_policy_test.exs | 6 ++ test/web/activity_pub/transmogrifier_test.exs | 4 + test/web/federator_test.exs | 2 +- test/web/twitter_api/util_controller_test.exs | 43 ++++---- test/web/websub/websub_test.exs | 2 +- 30 files changed, 402 insertions(+), 159 deletions(-) create mode 100644 lib/pleroma/workers/background_worker.ex create mode 100644 lib/pleroma/workers/helper.ex create mode 100644 lib/pleroma/workers/mailer.ex create mode 100644 lib/pleroma/workers/scheduled_activity_worker.ex create mode 100644 lib/pleroma/workers/transmogrifier.ex create mode 100644 lib/pleroma/workers/web_pusher.ex diff --git a/lib/pleroma/digest_email_worker.ex b/lib/pleroma/digest_email_worker.ex index 3b0e2bca6..6e44cc955 100644 --- a/lib/pleroma/digest_email_worker.ex +++ b/lib/pleroma/digest_email_worker.ex @@ -1,6 +1,11 @@ defmodule Pleroma.DigestEmailWorker do + alias Pleroma.Repo + alias Pleroma.Workers.Mailer, as: MailerWorker + import Ecto.Query + defdelegate worker_args(queue), to: Pleroma.Workers.Helper + def perform do config = Pleroma.Config.get([:email_notifications, :digest]) negative_interval = -Map.fetch!(config, :interval) @@ -15,7 +20,11 @@ defmodule Pleroma.DigestEmailWorker do select: u ) |> Pleroma.Repo.all() - |> Enum.each(&PleromaJobQueue.enqueue(:digest_emails, __MODULE__, [&1])) + |> Enum.each(fn user -> + %{"op" => "digest_email", "user_id" => user.id} + |> MailerWorker.new([queue: "digest_emails"] ++ worker_args(:digest_emails)) + |> Repo.insert() + end) end @doc """ diff --git a/lib/pleroma/scheduled_activity_worker.ex b/lib/pleroma/scheduled_activity_worker.ex index 65b38622f..cabea51ca 100644 --- a/lib/pleroma/scheduled_activity_worker.ex +++ b/lib/pleroma/scheduled_activity_worker.ex @@ -8,14 +8,18 @@ defmodule Pleroma.ScheduledActivityWorker do """ alias Pleroma.Config + alias Pleroma.Repo alias Pleroma.ScheduledActivity alias Pleroma.User alias Pleroma.Web.CommonAPI + use GenServer require Logger @schedule_interval :timer.minutes(1) + defdelegate worker_args(queue), to: Pleroma.Workers.Helper + def start_link do GenServer.start_link(__MODULE__, nil) end @@ -45,7 +49,9 @@ defmodule Pleroma.ScheduledActivityWorker do def handle_info(:perform, state) do ScheduledActivity.due_activities(@schedule_interval) |> Enum.each(fn scheduled_activity -> - PleromaJobQueue.enqueue(:scheduled_activities, __MODULE__, [:execute, scheduled_activity.id]) + %{"op" => "execute", "activity_id" => scheduled_activity.id} + |> Pleroma.Workers.ScheduledActivityWorker.new(worker_args(:scheduled_activities)) + |> Repo.insert() end) schedule_next() diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 7d18f099e..bc2102ca7 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -26,6 +26,7 @@ defmodule Pleroma.User do alias Pleroma.Web.OStatus alias Pleroma.Web.RelMe alias Pleroma.Web.Websub + alias Pleroma.Workers.BackgroundWorker require Logger @@ -39,6 +40,8 @@ defmodule Pleroma.User do @strict_local_nickname_regex ~r/^[a-zA-Z\d]+$/ @extended_local_nickname_regex ~r/^[a-zA-Z\d_-]+$/ + defdelegate worker_args(queue), to: Pleroma.Workers.Helper + schema "users" do field(:bio, :string) field(:email, :string) @@ -579,8 +582,11 @@ defmodule Pleroma.User do end @doc "Fetch some posts when the user has just been federated with" - def fetch_initial_posts(user), - do: PleromaJobQueue.enqueue(:background, __MODULE__, [:fetch_initial_posts, user]) + def fetch_initial_posts(user) do + %{"op" => "fetch_initial_posts", "user_id" => user.id} + |> BackgroundWorker.new(worker_args(:background)) + |> Repo.insert() + end @spec get_followers_query(User.t(), pos_integer() | nil) :: Ecto.Query.t() def get_followers_query(%User{} = user, nil) do @@ -1001,7 +1007,9 @@ defmodule Pleroma.User do end def deactivate_async(user, status \\ true) do - PleromaJobQueue.enqueue(:background, __MODULE__, [:deactivate_async, user, status]) + %{"op" => "deactivate_user", "user_id" => user.id, "status" => status} + |> BackgroundWorker.new(worker_args(:background)) + |> Repo.insert() end def deactivate(%User{} = user, status \\ true) do @@ -1029,9 +1037,11 @@ defmodule Pleroma.User do |> update_and_set_cache() end - @spec delete(User.t()) :: :ok - def delete(%User{} = user), - do: PleromaJobQueue.enqueue(:background, __MODULE__, [:delete, user]) + def delete(%User{} = user) do + %{"op" => "delete_user", "user_id" => user.id} + |> BackgroundWorker.new(worker_args(:background)) + |> Repo.insert() + end @spec perform(atom(), User.t()) :: {:ok, User.t()} def perform(:delete, %User{} = user) do @@ -1138,21 +1148,26 @@ defmodule Pleroma.User do Repo.all(query) end - def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers), - do: - PleromaJobQueue.enqueue(:background, __MODULE__, [ - :blocks_import, - blocker, - blocked_identifiers - ]) + def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers) do + %{ + "op" => "blocks_import", + "blocker_id" => blocker.id, + "blocked_identifiers" => blocked_identifiers + } + |> BackgroundWorker.new(worker_args(:background)) + |> Repo.insert() + end - def follow_import(%User{} = follower, followed_identifiers) when is_list(followed_identifiers), - do: - PleromaJobQueue.enqueue(:background, __MODULE__, [ - :follow_import, - follower, - followed_identifiers - ]) + def follow_import(%User{} = follower, followed_identifiers) + when is_list(followed_identifiers) do + %{ + "op" => "follow_import", + "follower_id" => follower.id, + "followed_identifiers" => followed_identifiers + } + |> BackgroundWorker.new(worker_args(:background)) + |> Repo.insert() + end def delete_user_activities(%User{ap_id: ap_id} = user) do ap_id diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 1a279a7df..8be8ac86f 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -17,6 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.WebFinger + alias Pleroma.Workers.BackgroundWorker import Ecto.Query import Pleroma.Web.ActivityPub.Utils @@ -25,6 +26,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do require Logger require Pleroma.Constants + defdelegate worker_args(queue), to: Pleroma.Workers.Helper + # For Announce activities, we filter the recipients based on following status for any actors # that match actual users. See issue #164 for more information about why this is necessary. defp get_recipients(%{"type" => "Announce"} = data) do @@ -145,7 +148,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do activity end - PleromaJobQueue.enqueue(:background, Pleroma.Web.RichMedia.Helpers, [:fetch, activity]) + %{"op" => "fetch_data_for_activity", "activity_id" => activity.id} + |> BackgroundWorker.new(worker_args(:background)) + |> Repo.insert() Notification.create_notifications(activity) diff --git a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex index 01d21a299..1df3bb5b6 100644 --- a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex @@ -7,7 +7,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do @behaviour Pleroma.Web.ActivityPub.MRF alias Pleroma.HTTP + alias Pleroma.Repo alias Pleroma.Web.MediaProxy + alias Pleroma.Workers.BackgroundWorker require Logger @@ -16,6 +18,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do recv_timeout: 10_000 ] + defdelegate worker_args(queue), to: Pleroma.Workers.Helper + def perform(:prefetch, url) do Logger.info("Prefetching #{inspect(url)}") @@ -30,7 +34,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do url |> Enum.each(fn %{"href" => href} -> - PleromaJobQueue.enqueue(:background, __MODULE__, [:prefetch, href]) + %{"op" => "media_proxy_prefetch", "url" => href} + |> BackgroundWorker.new(worker_args(:background)) + |> Repo.insert() x -> Logger.debug("Unhandled attachment URL object #{inspect(x)}") @@ -46,7 +52,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do %{"type" => "Create", "object" => %{"attachment" => attachments} = _object} = message ) when is_list(attachments) and length(attachments) > 0 do - PleromaJobQueue.enqueue(:background, __MODULE__, [:preload, message]) + %{"op" => "media_proxy_preload", "message" => message} + |> BackgroundWorker.new(worker_args(:background)) + |> Repo.insert() {:ok, message} end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 5403b71d8..0f117cd04 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -15,12 +15,15 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.Federator + alias Pleroma.Workers.Transmogrifier, as: TransmogrifierWorker import Ecto.Query require Logger require Pleroma.Constants + defdelegate worker_args(queue), to: Pleroma.Workers.Helper + @doc """ Modifies an incoming AP object (mastodon format) to our internal format. """ @@ -1073,7 +1076,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do already_ap <- User.ap_enabled?(user), {:ok, user} <- user |> User.upgrade_changeset(data) |> User.update_and_set_cache() do unless already_ap do - PleromaJobQueue.enqueue(:transmogrifier, __MODULE__, [:user_upgrade, user]) + %{"op" => "user_upgrade", "user_id" => user.id} + |> TransmogrifierWorker.new(worker_args(:transmogrifier)) + |> Repo.insert() end {:ok, user} diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index bb9eadfee..d85fe824f 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -3,12 +3,23 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Federator do + alias Pleroma.Activity + alias Pleroma.Object.Containment + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Web.Federator.Publisher + alias Pleroma.Web.OStatus + alias Pleroma.Web.Websub alias Pleroma.Workers.Publisher, as: PublisherWorker alias Pleroma.Workers.Receiver, as: ReceiverWorker alias Pleroma.Workers.Subscriber, as: SubscriberWorker require Logger + defdelegate worker_args(queue), to: Pleroma.Workers.Helper + def init do # 1 minute refresh_subscriptions(schedule_in: 60) @@ -41,7 +52,7 @@ defmodule Pleroma.Web.Federator do end def publish(%{id: "pleroma:fakeid"} = activity) do - PublisherWorker.perform_publish(activity) + perform(:publish, activity) end def publish(activity) do @@ -68,11 +79,88 @@ defmodule Pleroma.Web.Federator do |> Pleroma.Repo.insert() end - defp worker_args(queue) do - if max_attempts = Pleroma.Config.get([:workers, :retries, queue]) do - [max_attempts: max_attempts] + # Job Worker Callbacks + + @spec perform(atom(), module(), any()) :: {:ok, any()} | {:error, any()} + def perform(:publish_one, module, params) do + apply(module, :publish_one, [params]) + end + + def perform(:publish, activity) do + Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end) + + with %User{} = actor <- User.get_cached_by_ap_id(activity.data["actor"]), + {:ok, actor} <- User.ensure_keys_present(actor) do + Publisher.publish(actor, activity) + end + end + + def perform(:incoming_doc, doc) do + Logger.info("Got document, trying to parse") + OStatus.handle_incoming(doc) + end + + def perform(:incoming_ap_doc, params) do + Logger.info("Handling incoming AP activity") + + params = Utils.normalize_params(params) + + # NOTE: we use the actor ID to do the containment, this is fine because an + # actor shouldn't be acting on objects outside their own AP server. + with {:ok, _user} <- ap_enabled_actor(params["actor"]), + nil <- Activity.normalize(params["id"]), + :ok <- Containment.contain_origin_from_id(params["actor"], params), + {:ok, activity} <- Transmogrifier.handle_incoming(params) do + {:ok, activity} else - [] + %Activity{} -> + Logger.info("Already had #{params["id"]}") + :error + + _e -> + # Just drop those for now + Logger.info("Unhandled activity") + Logger.info(Jason.encode!(params, pretty: true)) + :error + end + end + + def perform(:request_subscription, websub) do + Logger.debug("Refreshing #{websub.topic}") + + with {:ok, websub} <- Websub.request_subscription(websub) do + Logger.debug("Successfully refreshed #{websub.topic}") + else + _e -> Logger.debug("Couldn't refresh #{websub.topic}") + end + end + + def perform(:verify_websub, websub) do + Logger.debug(fn -> + "Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})" + end) + + Websub.verify(websub) + end + + def perform(:refresh_subscriptions) do + Logger.debug("Federator running refresh subscriptions") + Websub.refresh_subscriptions() + + spawn(fn -> + # 6 hours + Process.sleep(1000 * 60 * 60 * 6) + refresh_subscriptions() + end) + end + + def ap_enabled_actor(id) do + user = User.get_cached_by_ap_id(id) + + if User.ap_enabled?(user) do + {:ok, user} + else + ActivityPub.make_user_from_ap_id(id) end end end diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex index dca852449..c0c9c3653 100644 --- a/lib/pleroma/web/oauth/token/clean_worker.ex +++ b/lib/pleroma/web/oauth/token/clean_worker.ex @@ -14,9 +14,12 @@ defmodule Pleroma.Web.OAuth.Token.CleanWorker do [:oauth2, :clean_expired_tokens_interval], 86_400_000 ) - @queue :background + alias Pleroma.Repo alias Pleroma.Web.OAuth.Token + alias Pleroma.Workers.BackgroundWorker + + defdelegate worker_args(queue), to: Pleroma.Workers.Helper def start_link, do: GenServer.start_link(__MODULE__, nil) @@ -31,8 +34,11 @@ defmodule Pleroma.Web.OAuth.Token.CleanWorker do @doc false def handle_info(:perform, state) do + %{"op" => "clean_expired_tokens"} + |> BackgroundWorker.new(worker_args(:background)) + |> Repo.insert() + Process.send_after(self(), :perform, @interval) - PleromaJobQueue.enqueue(@queue, __MODULE__, [:clean]) {:noreply, state} end diff --git a/lib/pleroma/web/push/push.ex b/lib/pleroma/web/push/push.ex index 729dad02a..b4f0e5127 100644 --- a/lib/pleroma/web/push/push.ex +++ b/lib/pleroma/web/push/push.ex @@ -3,10 +3,13 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Push do - alias Pleroma.Web.Push.Impl + alias Pleroma.Repo + alias Pleroma.Workers.WebPusher require Logger + defdelegate worker_args(queue), to: Pleroma.Workers.Helper + def init do unless enabled() do Logger.warn(""" @@ -31,6 +34,9 @@ defmodule Pleroma.Web.Push do end end - def send(notification), - do: PleromaJobQueue.enqueue(:web_push, Impl, [notification]) + def send(notification) do + %{"op" => "web_push", "notification_id" => notification.id} + |> WebPusher.new(worker_args(:web_push)) + |> Repo.insert() + end end diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 3405bd3b7..7ba4ad305 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -265,12 +265,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do String.split(line, ",") |> List.first() end) |> List.delete("Account address") do - PleromaJobQueue.enqueue(:background, User, [ - :follow_import, - follower, - followed_identifiers - ]) - + User.follow_import(follower, followed_identifiers) json(conn, "job started") end end @@ -281,12 +276,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do def blocks_import(%{assigns: %{user: blocker}} = conn, %{"list" => list}) do with blocked_identifiers <- String.split(list) do - PleromaJobQueue.enqueue(:background, User, [ - :blocks_import, - blocker, - blocked_identifiers - ]) - + User.blocks_import(blocker, blocked_identifiers) json(conn, "job started") end end diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex new file mode 100644 index 000000000..3ab2b6bcc --- /dev/null +++ b/lib/pleroma/workers/background_worker.ex @@ -0,0 +1,66 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.BackgroundWorker do + alias Pleroma.Activity + alias Pleroma.User + alias Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy + alias Pleroma.Web.OAuth.Token.CleanWorker + + # Note: `max_attempts` is intended to be overridden in `new/1` call + use Oban.Worker, + queue: "background", + max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + + @impl Oban.Worker + def perform(%{"op" => "fetch_initial_posts", "user_id" => user_id}) do + user = User.get_by_id(user_id) + User.perform(:fetch_initial_posts, user) + end + + def perform(%{"op" => "deactivate_user", "user_id" => user_id, "status" => status}) do + user = User.get_by_id(user_id) + User.perform(:deactivate_async, user, status) + end + + def perform(%{"op" => "delete_user", "user_id" => user_id}) do + user = User.get_by_id(user_id) + User.perform(:delete, user) + end + + def perform(%{ + "op" => "blocks_import", + "blocker_id" => blocker_id, + "blocked_identifiers" => blocked_identifiers + }) do + blocker = User.get_by_id(blocker_id) + User.perform(:blocks_import, blocker, blocked_identifiers) + end + + def perform(%{ + "op" => "follow_import", + "follower_id" => follower_id, + "followed_identifiers" => followed_identifiers + }) do + follower = User.get_by_id(follower_id) + User.perform(:follow_import, follower, followed_identifiers) + end + + def perform(%{"op" => "clean_expired_tokens"}) do + CleanWorker.perform(:clean) + end + + def perform(%{"op" => "media_proxy_preload", "message" => message}) do + MediaProxyWarmingPolicy.perform(:preload, message) + end + + def perform(%{"op" => "media_proxy_prefetch", "url" => url}) do + MediaProxyWarmingPolicy.perform(:prefetch, url) + end + + def perform(%{"op" => "fetch_data_for_activity", "activity_id" => activity_id}) do + activity = Activity.get_by_id(activity_id) + Pleroma.Web.RichMedia.Helpers.perform(:fetch, activity) + end +end diff --git a/lib/pleroma/workers/helper.ex b/lib/pleroma/workers/helper.ex new file mode 100644 index 000000000..3286ce0e8 --- /dev/null +++ b/lib/pleroma/workers/helper.ex @@ -0,0 +1,13 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Helper do + def worker_args(queue) do + if max_attempts = Pleroma.Config.get([:workers, :retries, queue]) do + [max_attempts: max_attempts] + else + [] + end + end +end diff --git a/lib/pleroma/workers/mailer.ex b/lib/pleroma/workers/mailer.ex new file mode 100644 index 000000000..da7fa6fd5 --- /dev/null +++ b/lib/pleroma/workers/mailer.ex @@ -0,0 +1,18 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Mailer do + alias Pleroma.User + + # Note: `max_attempts` is intended to be overridden in `new/1` call + use Oban.Worker, + queue: "mailer", + max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + + @impl Oban.Worker + def perform(%{"op" => "digest_email", "user_id" => user_id}) do + user = User.get_by_id(user_id) + Pleroma.DigestEmailWorker.perform(user) + end +end diff --git a/lib/pleroma/workers/publisher.ex b/lib/pleroma/workers/publisher.ex index 67871977a..c890ffb79 100644 --- a/lib/pleroma/workers/publisher.ex +++ b/lib/pleroma/workers/publisher.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Workers.Publisher do alias Pleroma.Activity - alias Pleroma.User + alias Pleroma.Web.Federator # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, @@ -13,23 +13,11 @@ defmodule Pleroma.Workers.Publisher do @impl Oban.Worker def perform(%{"op" => "publish", "activity_id" => activity_id}) do - with %Activity{} = activity <- Activity.get_by_id(activity_id) do - perform_publish(activity) - else - _ -> raise "Non-existing activity: #{activity_id}" - end + activity = Activity.get_by_id(activity_id) + Federator.perform(:publish, activity) end def perform(%{"op" => "publish_one", "module" => module_name, "params" => params}) do - module_name - |> String.to_atom() - |> apply(:publish_one, [params]) - end - - def perform_publish(%Activity{} = activity) do - with %User{} = actor <- User.get_cached_by_ap_id(activity.data["actor"]), - {:ok, actor} <- User.ensure_keys_present(actor) do - Pleroma.Web.Federator.Publisher.publish(actor, activity) - end + Federator.perform(:publish_one, String.to_atom(module_name), params) end end diff --git a/lib/pleroma/workers/receiver.ex b/lib/pleroma/workers/receiver.ex index 43558b4e6..d3de95716 100644 --- a/lib/pleroma/workers/receiver.ex +++ b/lib/pleroma/workers/receiver.ex @@ -3,15 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.Receiver do - alias Pleroma.Activity - alias Pleroma.Object.Containment - alias Pleroma.User - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.ActivityPub.Transmogrifier - alias Pleroma.Web.ActivityPub.Utils - alias Pleroma.Web.OStatus - - require Logger + alias Pleroma.Web.Federator # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, @@ -20,42 +12,10 @@ defmodule Pleroma.Workers.Receiver do @impl Oban.Worker def perform(%{"op" => "incoming_doc", "body" => doc}) do - Logger.info("Got incoming document, trying to parse") - OStatus.handle_incoming(doc) + Federator.perform(:incoming_doc, doc) end def perform(%{"op" => "incoming_ap_doc", "params" => params}) do - Logger.info("Handling incoming AP activity") - - params = Utils.normalize_params(params) - - # NOTE: we use the actor ID to do the containment, this is fine because an - # actor shouldn't be acting on objects outside their own AP server. - with {:ok, _user} <- ap_enabled_actor(params["actor"]), - nil <- Activity.normalize(params["id"]), - :ok <- Containment.contain_origin_from_id(params["actor"], params), - {:ok, activity} <- Transmogrifier.handle_incoming(params) do - {:ok, activity} - else - %Activity{} -> - Logger.info("Already had #{params["id"]}") - :error - - _e -> - # Just drop those for now - Logger.info("Unhandled activity") - Logger.info(Jason.encode!(params, pretty: true)) - :error - end - end - - defp ap_enabled_actor(id) do - user = User.get_cached_by_ap_id(id) - - if User.ap_enabled?(user) do - {:ok, user} - else - ActivityPub.make_user_from_ap_id(id) - end + Federator.perform(:incoming_ap_doc, params) end end diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex new file mode 100644 index 000000000..a49834fd8 --- /dev/null +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -0,0 +1,15 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.ScheduledActivityWorker do + # Note: `max_attempts` is intended to be overridden in `new/1` call + use Oban.Worker, + queue: "scheduled_activities", + max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + + @impl Oban.Worker + def perform(%{"op" => "execute", "activity_id" => activity_id}) do + Pleroma.ScheduledActivityWorker.perform(:execute, activity_id) + end +end diff --git a/lib/pleroma/workers/subscriber.ex b/lib/pleroma/workers/subscriber.ex index a8c01bb10..6af3ad0a1 100644 --- a/lib/pleroma/workers/subscriber.ex +++ b/lib/pleroma/workers/subscriber.ex @@ -4,11 +4,9 @@ defmodule Pleroma.Workers.Subscriber do alias Pleroma.Repo - alias Pleroma.Web.Websub + alias Pleroma.Web.Federator alias Pleroma.Web.Websub.WebsubClientSubscription - require Logger - # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, queue: "federator_outgoing", @@ -16,29 +14,16 @@ defmodule Pleroma.Workers.Subscriber do @impl Oban.Worker def perform(%{"op" => "refresh_subscriptions"}) do - Websub.refresh_subscriptions() - # Schedule the next run in 6 hours - Pleroma.Web.Federator.refresh_subscriptions(schedule_in: 3600 * 6) + Federator.perform(:refresh_subscriptions) end def perform(%{"op" => "request_subscription", "websub_id" => websub_id}) do websub = Repo.get(WebsubClientSubscription, websub_id) - Logger.debug("Refreshing #{websub.topic}") - - with {:ok, websub} <- Websub.request_subscription(websub) do - Logger.debug("Successfully refreshed #{websub.topic}") - else - _e -> Logger.debug("Couldn't refresh #{websub.topic}") - end + Federator.perform(:request_subscription, websub) end def perform(%{"op" => "verify_websub", "websub_id" => websub_id}) do websub = Repo.get(WebsubClientSubscription, websub_id) - - Logger.debug(fn -> - "Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})" - end) - - Websub.verify(websub) + Federator.perform(:verify_websub, websub) end end diff --git a/lib/pleroma/workers/transmogrifier.ex b/lib/pleroma/workers/transmogrifier.ex new file mode 100644 index 000000000..c6b4fab47 --- /dev/null +++ b/lib/pleroma/workers/transmogrifier.ex @@ -0,0 +1,18 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.Transmogrifier do + alias Pleroma.User + + # Note: `max_attempts` is intended to be overridden in `new/1` call + use Oban.Worker, + queue: "transmogrifier", + max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + + @impl Oban.Worker + def perform(%{"op" => "user_upgrade", "user_id" => user_id}) do + user = User.get_by_id(user_id) + Pleroma.Web.ActivityPub.Transmogrifier.perform(:user_upgrade, user) + end +end diff --git a/lib/pleroma/workers/web_pusher.ex b/lib/pleroma/workers/web_pusher.ex new file mode 100644 index 000000000..b99581eb0 --- /dev/null +++ b/lib/pleroma/workers/web_pusher.ex @@ -0,0 +1,19 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.WebPusher do + alias Pleroma.Notification + alias Pleroma.Repo + + # Note: `max_attempts` is intended to be overridden in `new/1` call + use Oban.Worker, + queue: "web_push", + max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + + @impl Oban.Worker + def perform(%{"op" => "web_push", "notification_id" => notification_id}) do + notification = Repo.get(Notification, notification_id) + Pleroma.Web.Push.Impl.perform(notification) + end +end diff --git a/test/activity_test.exs b/test/activity_test.exs index b9c12adb2..658c47837 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -6,8 +6,8 @@ defmodule Pleroma.ActivityTest do use Pleroma.DataCase alias Pleroma.Activity alias Pleroma.Bookmark - alias Pleroma.ObanHelpers alias Pleroma.Object + alias Pleroma.Tests.ObanHelpers alias Pleroma.ThreadMute import Pleroma.Factory diff --git a/test/conversation_test.exs b/test/conversation_test.exs index 2ebbcab76..f917aa691 100644 --- a/test/conversation_test.exs +++ b/test/conversation_test.exs @@ -28,7 +28,7 @@ defmodule Pleroma.ConversationTest do {:ok, _activity} = CommonAPI.post(user, %{"visibility" => "direct", "status" => "hey @#{other_user.nickname}"}) - Pleroma.ObanHelpers.perform_all() + Pleroma.Tests.ObanHelpers.perform_all() Repo.delete_all(Conversation) Repo.delete_all(Conversation.Participation) diff --git a/test/notification_test.exs b/test/notification_test.exs index 80ea2a085..e1c9f4f93 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.NotificationTest do import Pleroma.Factory alias Pleroma.Notification + alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.CommonAPI @@ -621,7 +622,8 @@ defmodule Pleroma.NotificationTest do refute Enum.empty?(Notification.for_user(other_user)) - User.delete(user) + {:ok, job} = User.delete(user) + ObanHelpers.perform(job) assert Enum.empty?(Notification.for_user(other_user)) end @@ -666,6 +668,7 @@ defmodule Pleroma.NotificationTest do } {:ok, _delete_activity} = Transmogrifier.handle_incoming(delete_user_message) + ObanHelpers.perform_all() assert Enum.empty?(Notification.for_user(local_user)) end diff --git a/test/support/oban_helpers.ex b/test/support/oban_helpers.ex index ecc03ba1a..d379c9ec7 100644 --- a/test/support/oban_helpers.ex +++ b/test/support/oban_helpers.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2018 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.ObanHelpers do +defmodule Pleroma.Tests.ObanHelpers do @moduledoc """ Oban test helpers. """ diff --git a/test/user_test.exs b/test/user_test.exs index 8617752d7..9c2117a0b 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -5,9 +5,9 @@ defmodule Pleroma.UserTest do alias Pleroma.Activity alias Pleroma.Builders.UserBuilder - alias Pleroma.ObanHelpers alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.CommonAPI @@ -676,7 +676,9 @@ defmodule Pleroma.UserTest do user3.nickname ] - result = User.follow_import(user1, identifiers) + {:ok, job} = User.follow_import(user1, identifiers) + result = ObanHelpers.perform(job) + assert is_list(result) assert result == [user2, user3] end @@ -887,7 +889,9 @@ defmodule Pleroma.UserTest do user3.nickname ] - result = User.blocks_import(user1, identifiers) + {:ok, job} = User.blocks_import(user1, identifiers) + result = ObanHelpers.perform(job) + assert is_list(result) assert result == [user2, user3] end @@ -1013,7 +1017,8 @@ defmodule Pleroma.UserTest do {:ok, like_two, _} = CommonAPI.favorite(activity.id, follower) {:ok, repeat, _} = CommonAPI.repeat(activity_two.id, user) - {:ok, _} = User.delete(user) + {:ok, job} = User.delete(user) + {:ok, _user} = ObanHelpers.perform(job) follower = User.get_cached_by_id(follower.id) @@ -1043,7 +1048,8 @@ defmodule Pleroma.UserTest do {:ok, follower} = User.get_or_fetch_by_ap_id("http://mastodon.example.org/users/admin") {:ok, _} = User.follow(follower, user) - {:ok, _user} = User.delete(user) + {:ok, job} = User.delete(user) + {:ok, _user} = ObanHelpers.perform(job) assert ObanHelpers.member?( %{ @@ -1100,7 +1106,8 @@ defmodule Pleroma.UserTest do test "User.delete() plugs any possible zombie objects" do user = insert(:user) - {:ok, _} = User.delete(user) + {:ok, job} = User.delete(user) + {:ok, _} = ObanHelpers.perform(job) {:ok, cached_user} = Cachex.get(:user_cache, "ap_id:#{user.ap_id}") diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index d7f0a8264..f46353fdd 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -9,8 +9,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do import Pleroma.Factory alias Pleroma.Activity alias Pleroma.Instances - alias Pleroma.ObanHelpers alias Pleroma.Object + alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.ActivityPub.ObjectView alias Pleroma.Web.ActivityPub.UserView diff --git a/test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs b/test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs index 372e789be..95a809d25 100644 --- a/test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs +++ b/test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicyTest do use Pleroma.DataCase alias Pleroma.HTTP + alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy import Mock @@ -24,6 +25,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicyTest do test "it prefetches media proxy URIs" do with_mock HTTP, get: fn _, _, _ -> {:ok, []} end do MediaProxyWarmingPolicy.filter(@message) + + ObanHelpers.perform_all() + # Performing jobs which has been just enqueued + ObanHelpers.perform_all() + assert called(HTTP.get(:_, :_, :_)) end end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index e7498e005..52f46c141 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do alias Pleroma.Object alias Pleroma.Object.Fetcher alias Pleroma.Repo + alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier @@ -563,6 +564,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do |> Poison.decode!() {:ok, _} = Transmogrifier.handle_incoming(data) + ObanHelpers.perform_all() refute User.get_cached_by_ap_id(ap_id) end @@ -1132,6 +1134,8 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert user.info.note_count == 1 {:ok, user} = Transmogrifier.upgrade_user_from_ap_id("https://niu.moe/users/rye") + ObanHelpers.perform_all() + assert user.info.ap_enabled assert user.info.note_count == 1 assert user.follower_address == "https://niu.moe/users/rye/followers" diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index e0be4342b..9ca341b6d 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Web.FederatorTest do alias Pleroma.Instances - alias Pleroma.ObanHelpers + alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.CommonAPI alias Pleroma.Web.Federator alias Pleroma.Workers.Publisher, as: PublisherWorker diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index 640579c09..e3f129f72 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -4,9 +4,11 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do use Pleroma.Web.ConnCase + use Oban.Testing, repo: Pleroma.Repo alias Pleroma.Notification alias Pleroma.Repo + alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.CommonAPI import Pleroma.Factory @@ -50,8 +52,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do {File, [], read!: fn "follow_list.txt" -> "Account address,Show boosts\n#{user2.ap_id},true" - end}, - {PleromaJobQueue, [:passthrough], []} + end} ]) do response = conn @@ -59,15 +60,16 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do |> post("/api/pleroma/follow_import", %{"list" => %Plug.Upload{path: "follow_list.txt"}}) |> json_response(:ok) - assert called( - PleromaJobQueue.enqueue( - :background, - User, - [:follow_import, user1, [user2.ap_id]] - ) - ) - assert response == "job started" + + assert ObanHelpers.member?( + %{ + "op" => "follow_import", + "follower_id" => user1.id, + "followed_identifiers" => [user2.ap_id] + }, + all_enqueued(worker: Pleroma.Workers.BackgroundWorker) + ) end end @@ -126,8 +128,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do user3 = insert(:user) with_mocks([ - {File, [], read!: fn "blocks_list.txt" -> "#{user2.ap_id} #{user3.ap_id}" end}, - {PleromaJobQueue, [:passthrough], []} + {File, [], read!: fn "blocks_list.txt" -> "#{user2.ap_id} #{user3.ap_id}" end} ]) do response = conn @@ -135,15 +136,16 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do |> post("/api/pleroma/blocks_import", %{"list" => %Plug.Upload{path: "blocks_list.txt"}}) |> json_response(:ok) - assert called( - PleromaJobQueue.enqueue( - :background, - User, - [:blocks_import, user1, [user2.ap_id, user3.ap_id]] - ) - ) - assert response == "job started" + + assert ObanHelpers.member?( + %{ + "op" => "blocks_import", + "blocker_id" => user1.id, + "blocked_identifiers" => [user2.ap_id, user3.ap_id] + }, + all_enqueued(worker: Pleroma.Workers.BackgroundWorker) + ) end end end @@ -607,6 +609,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do |> json_response(:ok) assert response == %{"status" => "success"} + ObanHelpers.perform_all() user = User.get_cached_by_id(user.id) diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index b704a558a..414610879 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -6,7 +6,7 @@ defmodule Pleroma.Web.WebsubTest do use Pleroma.DataCase use Oban.Testing, repo: Pleroma.Repo - alias Pleroma.ObanHelpers + alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.Router.Helpers alias Pleroma.Web.Websub alias Pleroma.Web.Websub.WebsubClientSubscription From a180c1360ecdbed76eccf3435bb2c831356746bc Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 14 Aug 2019 21:42:21 +0300 Subject: [PATCH 05/45] [#1149] Oban mailer job. Adjusted tests. --- lib/pleroma/application.ex | 1 + lib/pleroma/emails/mailer.ex | 13 ++++++++++++- lib/pleroma/workers/mailer.ex | 9 +++++++++ test/mix/tasks/pleroma.digest_test.exs | 3 +++ .../mastodon_api/mastodon_api_controller_test.exs | 4 ++++ .../web/twitter_api/twitter_api_controller_test.exs | 4 ++++ test/web/twitter_api/twitter_api_test.exs | 2 ++ 7 files changed, 35 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 5550a4902..7cf60f44a 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -233,6 +233,7 @@ defmodule Pleroma.Application do defp after_supervisor_start do with digest_config <- Application.get_env(:pleroma, :email_notifications)[:digest], true <- digest_config[:active] do + # TODO: consider replacing with `quantum` scheduler PleromaJobQueue.schedule( digest_config[:schedule], :digest_emails, diff --git a/lib/pleroma/emails/mailer.ex b/lib/pleroma/emails/mailer.ex index 2e4657b7c..bb534f602 100644 --- a/lib/pleroma/emails/mailer.ex +++ b/lib/pleroma/emails/mailer.ex @@ -9,6 +9,8 @@ defmodule Pleroma.Emails.Mailer do The module contains functions to delivery email using Swoosh.Mailer. """ + alias Pleroma.Repo + alias Pleroma.Workers.Mailer, as: MailerWorker alias Swoosh.DeliveryError @otp_app :pleroma @@ -17,9 +19,18 @@ defmodule Pleroma.Emails.Mailer do @spec enabled?() :: boolean() def enabled?, do: Pleroma.Config.get([__MODULE__, :enabled]) + defdelegate worker_args(queue), to: Pleroma.Workers.Helper + @doc "add email to queue" def deliver_async(email, config \\ []) do - PleromaJobQueue.enqueue(:mailer, __MODULE__, [:deliver_async, email, config]) + encoded_email = + email + |> :erlang.term_to_binary() + |> Base.encode64() + + %{"op" => "email", "encoded_email" => encoded_email, "config" => config} + |> MailerWorker.new(worker_args(:mailer)) + |> Repo.insert() end @doc "callback to perform send email from queue" diff --git a/lib/pleroma/workers/mailer.ex b/lib/pleroma/workers/mailer.ex index da7fa6fd5..8bf9952bc 100644 --- a/lib/pleroma/workers/mailer.ex +++ b/lib/pleroma/workers/mailer.ex @@ -11,6 +11,15 @@ defmodule Pleroma.Workers.Mailer do max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) @impl Oban.Worker + def perform(%{"op" => "email", "encoded_email" => encoded_email, "config" => config}) do + email = + encoded_email + |> Base.decode64!() + |> :erlang.binary_to_term() + + Pleroma.Emails.Mailer.deliver(email, config) + end + def perform(%{"op" => "digest_email", "user_id" => user_id}) do user = User.get_by_id(user_id) Pleroma.DigestEmailWorker.perform(user) diff --git a/test/mix/tasks/pleroma.digest_test.exs b/test/mix/tasks/pleroma.digest_test.exs index 595f64ed7..5fbeac0d6 100644 --- a/test/mix/tasks/pleroma.digest_test.exs +++ b/test/mix/tasks/pleroma.digest_test.exs @@ -4,6 +4,7 @@ defmodule Mix.Tasks.Pleroma.DigestTest do import Pleroma.Factory import Swoosh.TestAssertions + alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.CommonAPI setup_all do @@ -39,6 +40,8 @@ defmodule Mix.Tasks.Pleroma.DigestTest do :ok = Mix.Tasks.Pleroma.Digest.run(["test", user2.nickname, yesterday_date]) + ObanHelpers.perform_all() + assert_receive {:mix_shell, :info, [message]} assert message =~ "Digest email have been sent" diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index e49c4cc22..be9ff2568 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -11,6 +11,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do alias Pleroma.Object alias Pleroma.Repo alias Pleroma.ScheduledActivity + alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.CommonAPI @@ -3871,6 +3872,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do end test "it sends an email to user", %{user: user} do + ObanHelpers.perform_all() token_record = Repo.get_by(Pleroma.PasswordResetToken, user_id: user.id) email = Pleroma.Emails.UserEmail.password_reset_email(user, token_record.token) @@ -3934,6 +3936,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do |> post("/api/v1/pleroma/accounts/confirmation_resend?email=#{user.email}") |> json_response(:no_content) + ObanHelpers.perform_all() + email = Pleroma.Emails.UserEmail.account_confirmation_email(user) notify_email = Pleroma.Config.get([:instance, :notify_email]) instance_name = Pleroma.Config.get([:instance, :name]) diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index 8bb8aa36d..9ac4ff929 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -12,6 +12,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.CommonAPI @@ -1099,6 +1100,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do end test "it sends an email to user", %{user: user} do + ObanHelpers.perform_all() token_record = Repo.get_by(Pleroma.PasswordResetToken, user_id: user.id) email = Pleroma.Emails.UserEmail.password_reset_email(user, token_record.token) @@ -1209,6 +1211,8 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do |> assign(:user, user) |> post("/api/account/resend_confirmation_email?email=#{user.email}") + ObanHelpers.perform_all() + email = Pleroma.Emails.UserEmail.account_confirmation_email(user) notify_email = Pleroma.Config.get([:instance, :notify_email]) instance_name = Pleroma.Config.get([:instance, :name]) diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index cbe83852e..bf063a0de 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPITest do alias Pleroma.Activity alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.UserInviteToken alias Pleroma.Web.ActivityPub.ActivityPub @@ -321,6 +322,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPITest do } {:ok, user} = TwitterAPI.register_user(data) + ObanHelpers.perform_all() assert user.info.confirmation_pending From c29686309eaf2cdae039ce813755c0e23cdc4a03 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 23 Aug 2019 09:23:10 +0300 Subject: [PATCH 06/45] [#1149] Upgraded `oban` from 0.6.0 to 0.7.1. --- config/config.exs | 1 - lib/pleroma/application.ex | 5 +-- lib/pleroma/workers/background_worker.ex | 42 +++++++++++-------- lib/pleroma/workers/mailer.ex | 6 +-- lib/pleroma/workers/publisher.ex | 6 +-- lib/pleroma/workers/receiver.ex | 6 +-- .../workers/scheduled_activity_worker.ex | 4 +- lib/pleroma/workers/subscriber.ex | 8 ++-- lib/pleroma/workers/transmogrifier.ex | 4 +- lib/pleroma/workers/web_pusher.ex | 4 +- mix.exs | 2 +- mix.lock | 10 ++--- test/support/oban_helpers.ex | 2 +- 13 files changed, 51 insertions(+), 49 deletions(-) diff --git a/config/config.exs b/config/config.exs index 9794997d9..1a6348bcd 100644 --- a/config/config.exs +++ b/config/config.exs @@ -469,7 +469,6 @@ config :pleroma, Oban, config :pleroma, :workers, retries: [ - compile_time_default: 1, federator_incoming: 5, federator_outgoing: 5 ] diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 2e2922d28..384b03aa9 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -41,10 +41,7 @@ defmodule Pleroma.Application do hackney_pool_children() ++ [ Pleroma.Stats, - %{ - id: Oban, - start: {Oban, :start_link, [Application.get_env(:pleroma, Oban)]} - }, + {Oban, Application.get_env(:pleroma, Oban)}, %{ id: :web_push_init, start: {Task, :start_link, [&Pleroma.Web.Push.init/0]}, diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index 3ab2b6bcc..3c021b9b4 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -11,55 +11,61 @@ defmodule Pleroma.Workers.BackgroundWorker do # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, queue: "background", - max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + max_attempts: 1 @impl Oban.Worker - def perform(%{"op" => "fetch_initial_posts", "user_id" => user_id}) do + def perform(%{"op" => "fetch_initial_posts", "user_id" => user_id}, _job) do user = User.get_by_id(user_id) User.perform(:fetch_initial_posts, user) end - def perform(%{"op" => "deactivate_user", "user_id" => user_id, "status" => status}) do + def perform(%{"op" => "deactivate_user", "user_id" => user_id, "status" => status}, _job) do user = User.get_by_id(user_id) User.perform(:deactivate_async, user, status) end - def perform(%{"op" => "delete_user", "user_id" => user_id}) do + def perform(%{"op" => "delete_user", "user_id" => user_id}, _job) do user = User.get_by_id(user_id) User.perform(:delete, user) end - def perform(%{ - "op" => "blocks_import", - "blocker_id" => blocker_id, - "blocked_identifiers" => blocked_identifiers - }) do + def perform( + %{ + "op" => "blocks_import", + "blocker_id" => blocker_id, + "blocked_identifiers" => blocked_identifiers + }, + _job + ) do blocker = User.get_by_id(blocker_id) User.perform(:blocks_import, blocker, blocked_identifiers) end - def perform(%{ - "op" => "follow_import", - "follower_id" => follower_id, - "followed_identifiers" => followed_identifiers - }) do + def perform( + %{ + "op" => "follow_import", + "follower_id" => follower_id, + "followed_identifiers" => followed_identifiers + }, + _job + ) do follower = User.get_by_id(follower_id) User.perform(:follow_import, follower, followed_identifiers) end - def perform(%{"op" => "clean_expired_tokens"}) do + def perform(%{"op" => "clean_expired_tokens"}, _job) do CleanWorker.perform(:clean) end - def perform(%{"op" => "media_proxy_preload", "message" => message}) do + def perform(%{"op" => "media_proxy_preload", "message" => message}, _job) do MediaProxyWarmingPolicy.perform(:preload, message) end - def perform(%{"op" => "media_proxy_prefetch", "url" => url}) do + def perform(%{"op" => "media_proxy_prefetch", "url" => url}, _job) do MediaProxyWarmingPolicy.perform(:prefetch, url) end - def perform(%{"op" => "fetch_data_for_activity", "activity_id" => activity_id}) do + def perform(%{"op" => "fetch_data_for_activity", "activity_id" => activity_id}, _job) do activity = Activity.get_by_id(activity_id) Pleroma.Web.RichMedia.Helpers.perform(:fetch, activity) end diff --git a/lib/pleroma/workers/mailer.ex b/lib/pleroma/workers/mailer.ex index 8bf9952bc..1cce2ea03 100644 --- a/lib/pleroma/workers/mailer.ex +++ b/lib/pleroma/workers/mailer.ex @@ -8,10 +8,10 @@ defmodule Pleroma.Workers.Mailer do # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, queue: "mailer", - max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + max_attempts: 1 @impl Oban.Worker - def perform(%{"op" => "email", "encoded_email" => encoded_email, "config" => config}) do + def perform(%{"op" => "email", "encoded_email" => encoded_email, "config" => config}, _job) do email = encoded_email |> Base.decode64!() @@ -20,7 +20,7 @@ defmodule Pleroma.Workers.Mailer do Pleroma.Emails.Mailer.deliver(email, config) end - def perform(%{"op" => "digest_email", "user_id" => user_id}) do + def perform(%{"op" => "digest_email", "user_id" => user_id}, _job) do user = User.get_by_id(user_id) Pleroma.DigestEmailWorker.perform(user) end diff --git a/lib/pleroma/workers/publisher.ex b/lib/pleroma/workers/publisher.ex index c890ffb79..0a9084589 100644 --- a/lib/pleroma/workers/publisher.ex +++ b/lib/pleroma/workers/publisher.ex @@ -9,15 +9,15 @@ defmodule Pleroma.Workers.Publisher do # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, queue: "federator_outgoing", - max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + max_attempts: 1 @impl Oban.Worker - def perform(%{"op" => "publish", "activity_id" => activity_id}) do + def perform(%{"op" => "publish", "activity_id" => activity_id}, _job) do activity = Activity.get_by_id(activity_id) Federator.perform(:publish, activity) end - def perform(%{"op" => "publish_one", "module" => module_name, "params" => params}) do + def perform(%{"op" => "publish_one", "module" => module_name, "params" => params}, _job) do Federator.perform(:publish_one, String.to_atom(module_name), params) end end diff --git a/lib/pleroma/workers/receiver.ex b/lib/pleroma/workers/receiver.ex index d3de95716..4ee270d74 100644 --- a/lib/pleroma/workers/receiver.ex +++ b/lib/pleroma/workers/receiver.ex @@ -8,14 +8,14 @@ defmodule Pleroma.Workers.Receiver do # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, queue: "federator_incoming", - max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + max_attempts: 1 @impl Oban.Worker - def perform(%{"op" => "incoming_doc", "body" => doc}) do + def perform(%{"op" => "incoming_doc", "body" => doc}, _job) do Federator.perform(:incoming_doc, doc) end - def perform(%{"op" => "incoming_ap_doc", "params" => params}) do + def perform(%{"op" => "incoming_ap_doc", "params" => params}, _job) do Federator.perform(:incoming_ap_doc, params) end end diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex index a49834fd8..d9724c78a 100644 --- a/lib/pleroma/workers/scheduled_activity_worker.ex +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -6,10 +6,10 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, queue: "scheduled_activities", - max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + max_attempts: 1 @impl Oban.Worker - def perform(%{"op" => "execute", "activity_id" => activity_id}) do + def perform(%{"op" => "execute", "activity_id" => activity_id}, _job) do Pleroma.ScheduledActivityWorker.perform(:execute, activity_id) end end diff --git a/lib/pleroma/workers/subscriber.ex b/lib/pleroma/workers/subscriber.ex index 6af3ad0a1..783c44173 100644 --- a/lib/pleroma/workers/subscriber.ex +++ b/lib/pleroma/workers/subscriber.ex @@ -10,19 +10,19 @@ defmodule Pleroma.Workers.Subscriber do # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, queue: "federator_outgoing", - max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + max_attempts: 1 @impl Oban.Worker - def perform(%{"op" => "refresh_subscriptions"}) do + def perform(%{"op" => "refresh_subscriptions"}, _job) do Federator.perform(:refresh_subscriptions) end - def perform(%{"op" => "request_subscription", "websub_id" => websub_id}) do + def perform(%{"op" => "request_subscription", "websub_id" => websub_id}, _job) do websub = Repo.get(WebsubClientSubscription, websub_id) Federator.perform(:request_subscription, websub) end - def perform(%{"op" => "verify_websub", "websub_id" => websub_id}) do + def perform(%{"op" => "verify_websub", "websub_id" => websub_id}, _job) do websub = Repo.get(WebsubClientSubscription, websub_id) Federator.perform(:verify_websub, websub) end diff --git a/lib/pleroma/workers/transmogrifier.ex b/lib/pleroma/workers/transmogrifier.ex index c6b4fab47..e13202c06 100644 --- a/lib/pleroma/workers/transmogrifier.ex +++ b/lib/pleroma/workers/transmogrifier.ex @@ -8,10 +8,10 @@ defmodule Pleroma.Workers.Transmogrifier do # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, queue: "transmogrifier", - max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + max_attempts: 1 @impl Oban.Worker - def perform(%{"op" => "user_upgrade", "user_id" => user_id}) do + def perform(%{"op" => "user_upgrade", "user_id" => user_id}, _job) do user = User.get_by_id(user_id) Pleroma.Web.ActivityPub.Transmogrifier.perform(:user_upgrade, user) end diff --git a/lib/pleroma/workers/web_pusher.ex b/lib/pleroma/workers/web_pusher.ex index b99581eb0..7b78bb3ea 100644 --- a/lib/pleroma/workers/web_pusher.ex +++ b/lib/pleroma/workers/web_pusher.ex @@ -9,10 +9,10 @@ defmodule Pleroma.Workers.WebPusher do # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, queue: "web_push", - max_attempts: Pleroma.Config.get([:workers, :retries, :compile_time_default]) + max_attempts: 1 @impl Oban.Worker - def perform(%{"op" => "web_push", "notification_id" => notification_id}) do + def perform(%{"op" => "web_push", "notification_id" => notification_id}, _job) do notification = Repo.get(Notification, notification_id) Pleroma.Web.Push.Impl.perform(notification) end diff --git a/mix.exs b/mix.exs index b651520ed..eb023313d 100644 --- a/mix.exs +++ b/mix.exs @@ -101,7 +101,7 @@ defmodule Pleroma.Mixfile do {:phoenix_ecto, "~> 4.0"}, {:ecto_sql, "~> 3.1"}, {:postgrex, ">= 0.13.5"}, - {:oban, "~> 0.6"}, + {:oban, "~> 0.7"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, {:pbkdf2_elixir, "~> 0.12.3"}, diff --git a/mix.lock b/mix.lock index 52932c9ef..8b8596375 100644 --- a/mix.lock +++ b/mix.lock @@ -17,12 +17,12 @@ "credo": {:hex, :credo, "0.9.3", "76fa3e9e497ab282e0cf64b98a624aa11da702854c52c82db1bf24e54ab7c97a", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, "crontab": {:hex, :crontab, "1.1.7", "b9219f0bdc8678b94143655a8f229716c5810c0636a4489f98c0956137e53985", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]}, - "db_connection": {:hex, :db_connection, "2.0.6", "bde2f85d047969c5b5800cb8f4b3ed6316c8cb11487afedac4aa5f93fd39abfa", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, + "db_connection": {:hex, :db_connection, "2.1.1", "a51e8a2ee54ef2ae6ec41a668c85787ed40cb8944928c191280fe34c15b76ae5", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, "decimal": {:hex, :decimal, "1.8.0", "ca462e0d885f09a1c5a342dbd7c1dcf27ea63548c65a65e67334f4b61803822e", [:mix], [], "hexpm"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"}, - "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, - "ecto_sql": {:hex, :ecto_sql, "3.1.3", "2c536139190492d9de33c5fefac7323c5eaaa82e1b9bf93482a14649042f7cd9", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "ecto": {:hex, :ecto, "3.1.7", "fa21d06ef56cdc2fdaa62574e8c3ba34a2751d44ea34c30bc65f0728421043e5", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "ecto_sql": {:hex, :ecto_sql, "3.1.6", "1e80e30d16138a729c717f73dcb938590bcdb3a4502f3012414d0cbb261045d8", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0 or ~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"}, "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"}, "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"}, @@ -57,7 +57,7 @@ "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm"}, "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm"}, "nimble_parsec": {:hex, :nimble_parsec, "0.5.0", "90e2eca3d0266e5c53f8fbe0079694740b9c91b6747f2b7e3c5d21966bba8300", [:mix], [], "hexpm"}, - "oban": {:hex, :oban, "0.6.0", "8b9b861355610e703e58a878bc29959f3f0e1b4cd1e90d785cf2bb2498d3b893", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "oban": {:hex, :oban, "0.7.1", "171bdd1b69c1a4a839f8c768f5e962fc22d1de1513d459fb6b8e0cbd34817a9a", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.3", "6706a148809a29c306062862c803406e88f048277f6e85b68faf73291e820b84", [:mix], [], "hexpm"}, "phoenix": {:hex, :phoenix, "1.4.9", "746d098e10741c334d88143d3c94cab1756435f94387a63441792e66ec0ee974", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, @@ -71,7 +71,7 @@ "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"}, "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, - "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "postgrex": {:hex, :postgrex, "0.15.0", "dd5349161019caeea93efa42f9b22f9d79995c3a86bdffb796427b4c9863b0f0", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "prometheus": {:hex, :prometheus, "4.4.1", "1e96073b3ed7788053768fea779cbc896ddc3bdd9ba60687f2ad50b252ac87d6", [:mix, :rebar3], [], "hexpm"}, "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/support/oban_helpers.ex b/test/support/oban_helpers.ex index d379c9ec7..989770926 100644 --- a/test/support/oban_helpers.ex +++ b/test/support/oban_helpers.ex @@ -16,7 +16,7 @@ defmodule Pleroma.Tests.ObanHelpers do end def perform(%Oban.Job{} = job) do - res = apply(String.to_existing_atom("Elixir." <> job.worker), :perform, [job]) + res = apply(String.to_existing_atom("Elixir." <> job.worker), :perform, [job.args, job]) Repo.delete(job) res end From c056736daaedb2a08557ee6c6a9bcb6bf44110ca Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 23 Aug 2019 16:11:39 +0300 Subject: [PATCH 07/45] [#1149] Publisher worker fix (atomized `params` keys). --- lib/pleroma/workers/publisher.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/workers/publisher.ex b/lib/pleroma/workers/publisher.ex index 0a9084589..00fae99c7 100644 --- a/lib/pleroma/workers/publisher.ex +++ b/lib/pleroma/workers/publisher.ex @@ -18,6 +18,7 @@ defmodule Pleroma.Workers.Publisher do end def perform(%{"op" => "publish_one", "module" => module_name, "params" => params}, _job) do + params = Map.new(params, fn {k, v} -> {String.to_atom(k), v} end) Federator.perform(:publish_one, String.to_atom(module_name), params) end end From 581123f8bb703023cb652267a1fc34292f862852 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 23 Aug 2019 18:28:23 +0300 Subject: [PATCH 08/45] [#1149] Introduced `quantum` job scheduler. Documentation & config changes. --- CHANGELOG.md | 2 ++ config/config.exs | 40 +++++++++++++++++--------- config/test.exs | 2 -- docs/config.md | 15 ++++++---- lib/pleroma/application.ex | 19 ++---------- lib/pleroma/scheduler.ex | 7 +++++ lib/pleroma/web/federator/federator.ex | 8 +----- mix.exs | 2 +- mix.lock | 6 +++- 9 files changed, 54 insertions(+), 47 deletions(-) create mode 100644 lib/pleroma/scheduler.ex diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b0f4f40e..6dc19e79f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Unsubscribe followers when they unfollow a user - AdminAPI: Add "godmode" while fetching user statuses (i.e. admin can see private statuses) - Improve digest email template +- Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) with [Oban](https://github.com/sorentwo/oban) +- Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler ### Fixed - Not being able to pin unlisted posts diff --git a/config/config.exs b/config/config.exs index 1a6348bcd..43d114d70 100644 --- a/config/config.exs +++ b/config/config.exs @@ -51,6 +51,24 @@ config :pleroma, Pleroma.Repo, telemetry_event: [Pleroma.Repo.Instrumenter], migration_lock: nil +scheduled_jobs = + with digest_config <- Application.get_env(:pleroma, :email_notifications)[:digest], + true <- digest_config[:active] do + [{digest_config[:schedule], {Pleroma.DigestEmailWorker, :perform, []}}] + else + _ -> [] + end + +scheduled_jobs = + scheduled_jobs ++ + [{"0 */6 * * * *", {Pleroma.Web.Websub, :refresh_subscriptions, []}}] + +config :pleroma, Pleroma.Scheduler, + global: true, + overlap: true, + timezone: :utc, + jobs: scheduled_jobs + config :pleroma, Pleroma.Captcha, enabled: false, seconds_valid: 60, @@ -449,23 +467,19 @@ config :pleroma, Pleroma.User, "web" ] -job_queues = [ - federator_incoming: 50, - federator_outgoing: 50, - web_push: 50, - mailer: 10, - transmogrifier: 20, - scheduled_activities: 10, - background: 5 -] - -config :pleroma_job_queue, :queues, job_queues - config :pleroma, Oban, repo: Pleroma.Repo, verbose: false, prune: {:maxage, 60 * 60 * 24 * 7}, - queues: job_queues + queues: [ + federator_incoming: 50, + federator_outgoing: 50, + web_push: 50, + mailer: 10, + transmogrifier: 20, + scheduled_activities: 10, + background: 5 + ] config :pleroma, :workers, retries: [ diff --git a/config/test.exs b/config/test.exs index a0fa67516..62f2a04d2 100644 --- a/config/test.exs +++ b/config/test.exs @@ -61,8 +61,6 @@ config :web_push_encryption, :vapid_details, config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock -config :pleroma_job_queue, disabled: true - config :pleroma, Oban, queues: false, prune: :disabled diff --git a/docs/config.md b/docs/config.md index ae8afad89..81923c640 100644 --- a/docs/config.md +++ b/docs/config.md @@ -400,9 +400,9 @@ You can then do curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerandomtoken" ``` -## :pleroma_job_queue +## Oban -[Pleroma Job Queue](https://git.pleroma.social/pleroma/pleroma_job_queue) configuration: a list of queues with maximum concurrent jobs. +[Oban](https://github.com/sorentwo/oban) asynchronous job processor configuration. Pleroma has the following queues: @@ -416,12 +416,15 @@ Pleroma has the following queues: Example: ```elixir -config :pleroma_job_queue, :queues, - federator_incoming: 50, - federator_outgoing: 50 +config :pleroma, Oban, + repo: Pleroma.Repo, + queues: [ + federator_incoming: 50, + federator_outgoing: 50 + ] ``` -This config contains two queues: `federator_incoming` and `federator_outgoing`. Both have the `max_jobs` set to `50`. +This config contains two queues: `federator_incoming` and `federator_outgoing`. Both have the number of max concurrent jobs set to `50`. ## Pleroma.Web.Metadata * `providers`: a list of metadata providers to enable. Providers available: diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 384b03aa9..ce2d3ab59 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -31,6 +31,7 @@ defmodule Pleroma.Application do children = [ Pleroma.Repo, + Pleroma.Scheduler, Pleroma.Config.TransferTask, Pleroma.Emoji, Pleroma.Captcha, @@ -69,9 +70,7 @@ defmodule Pleroma.Application do # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # for other strategies and supported options opts = [strategy: :one_for_one, name: Pleroma.Supervisor] - result = Supervisor.start_link(children, opts) - :ok = after_supervisor_start() - result + Supervisor.start_link(children, opts) end defp setup_instrumenters do @@ -162,18 +161,4 @@ defmodule Pleroma.Application do :hackney_pool.child_spec(pool, options) end end - - defp after_supervisor_start do - with digest_config <- Application.get_env(:pleroma, :email_notifications)[:digest], - true <- digest_config[:active] do - # TODO: consider replacing with `quantum` scheduler - PleromaJobQueue.schedule( - digest_config[:schedule], - :digest_emails, - Pleroma.DigestEmailWorker - ) - end - - :ok - end end diff --git a/lib/pleroma/scheduler.ex b/lib/pleroma/scheduler.ex new file mode 100644 index 000000000..d84cd99ad --- /dev/null +++ b/lib/pleroma/scheduler.ex @@ -0,0 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Scheduler do + use Quantum.Scheduler, otp_app: :pleroma +end diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index d85fe824f..cf7e50fee 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -21,7 +21,7 @@ defmodule Pleroma.Web.Federator do defdelegate worker_args(queue), to: Pleroma.Workers.Helper def init do - # 1 minute + # To do: consider removing this call in favor of scheduled execution (`quantum`-based) refresh_subscriptions(schedule_in: 60) end @@ -146,12 +146,6 @@ defmodule Pleroma.Web.Federator do def perform(:refresh_subscriptions) do Logger.debug("Federator running refresh subscriptions") Websub.refresh_subscriptions() - - spawn(fn -> - # 6 hours - Process.sleep(1000 * 60 * 60 * 6) - refresh_subscriptions() - end) end def ap_enabled_actor(id) do diff --git a/mix.exs b/mix.exs index eb023313d..9d8ded1ff 100644 --- a/mix.exs +++ b/mix.exs @@ -102,6 +102,7 @@ defmodule Pleroma.Mixfile do {:ecto_sql, "~> 3.1"}, {:postgrex, ">= 0.13.5"}, {:oban, "~> 0.7"}, + {:quantum, "~> 2.3"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, {:pbkdf2_elixir, "~> 0.12.3"}, @@ -142,7 +143,6 @@ defmodule Pleroma.Mixfile do {:http_signatures, git: "https://git.pleroma.social/pleroma/http_signatures.git", ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"}, - {:pleroma_job_queue, "~> 0.3"}, {:telemetry, "~> 0.3"}, {:prometheus_ex, "~> 3.0"}, {:prometheus_plugs, "~> 1.1"}, diff --git a/mix.lock b/mix.lock index 8b8596375..6ebc66271 100644 --- a/mix.lock +++ b/mix.lock @@ -36,6 +36,8 @@ "excoveralls": {:hex, :excoveralls, "0.11.1", "dd677fbdd49114fdbdbf445540ec735808250d56b011077798316505064edb2c", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, "floki": {:hex, :floki, "0.20.4", "be42ac911fece24b4c72f3b5846774b6e61b83fe685c2fc9d62093277fb3bc86", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}, {:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, "gen_smtp": {:hex, :gen_smtp, "0.14.0", "39846a03522456077c6429b4badfd1d55e5e7d0fdfb65e935b7c5e38549d9202", [:rebar3], [], "hexpm"}, + "gen_stage": {:hex, :gen_stage, "0.14.2", "6a2a578a510c5bfca8a45e6b27552f613b41cf584b58210f017088d3d17d0b14", [:mix], [], "hexpm"}, + "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"}, "gettext": {:hex, :gettext, "0.17.0", "abe21542c831887a2b16f4c94556db9c421ab301aee417b7c4fbde7fbdbe01ec", [:mix], [], "hexpm"}, "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"}, @@ -46,6 +48,7 @@ "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, "joken": {:hex, :joken, "2.0.1", "ec9ab31bf660f343380da033b3316855197c8d4c6ef597fa3fcb451b326beb14", [:mix], [{:jose, "~> 1.9", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm"}, "jose": {:hex, :jose, "1.9.0", "4167c5f6d06ffaebffd15cdb8da61a108445ef5e85ab8f5a7ad926fdf3ada154", [:mix, :rebar3], [{:base64url, "~> 0.0.1", [hex: :base64url, repo: "hexpm", optional: false]}], "hexpm"}, + "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"}, "makeup": {:hex, :makeup, "0.8.0", "9cf32aea71c7fe0a4b2e9246c2c4978f9070257e5c9ce6d4a28ec450a839b55f", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, "makeup_elixir": {:hex, :makeup_elixir, "0.13.0", "be7a477997dcac2e48a9d695ec730b2d22418292675c75aa2d34ba0909dcdeda", [:mix], [{:makeup, "~> 0.8", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"}, "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm"}, @@ -65,7 +68,6 @@ "phoenix_html": {:hex, :phoenix_html, "2.13.1", "fa8f034b5328e2dfa0e4131b5569379003f34bc1fafdaa84985b0b9d2f12e68b", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "1.1.2", "496c303bdf1b2e98a9d26e89af5bba3ab487ba3a3735f74bf1f4064d2a845a3e", [:mix], [], "hexpm"}, "phoenix_swoosh": {:hex, :phoenix_swoosh, "0.2.0", "a7e0b32077cd6d2323ae15198839b05d9caddfa20663fd85787479e81f89520e", [:mix], [{:phoenix, "~> 1.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.2", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:swoosh, "~> 0.1", [hex: :swoosh, repo: "hexpm", optional: false]}], "hexpm"}, - "pleroma_job_queue": {:hex, :pleroma_job_queue, "0.3.0", "b84538d621f0c3d6fcc1cff9d5648d3faaf873b8b21b94e6503428a07a48ec47", [:mix], [{:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}], "hexpm"}, "plug": {:hex, :plug, "1.8.2", "0bcce1daa420f189a6491f3940cc77ea7fb1919761175c9c3b59800d897440fc", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm"}, "plug_cowboy": {:hex, :plug_cowboy, "2.1.0", "b75768153c3a8a9e8039d4b25bb9b14efbc58e9c4a6e6a270abff1cd30cbe320", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"}, @@ -78,9 +80,11 @@ "prometheus_phoenix": {:hex, :prometheus_phoenix, "1.3.0", "c4b527e0b3a9ef1af26bdcfbfad3998f37795b9185d475ca610fe4388fdd3bb5", [:mix], [{:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm"}, "quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm"}, + "quantum": {:hex, :quantum, "2.3.4", "72a0e8855e2adc101459eac8454787cb74ab4169de6ca50f670e72142d4960e9", [:mix], [{:calendar, "~> 0.17", [hex: :calendar, repo: "hexpm", optional: true]}, {:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.12", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:swarm, "~> 3.3", [hex: :swarm, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: true]}], "hexpm"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, + "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm"}, "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, From 71700ea6d4104ecd2cc0afb0ac103e722b30fbb5 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 24 Aug 2019 09:27:32 +0300 Subject: [PATCH 09/45] [#1149] Updated docs & tests. --- docs/config.md | 6 ++++++ test/web/admin_api/admin_api_controller_test.exs | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/config.md b/docs/config.md index 81923c640..5b2c3a022 100644 --- a/docs/config.md +++ b/docs/config.md @@ -426,6 +426,12 @@ config :pleroma, Oban, This config contains two queues: `federator_incoming` and `federator_outgoing`. Both have the number of max concurrent jobs set to `50`. +## :workers + +Includes custom worker options not interpretable directly by `Oban`. + +* `retries` — keyword lists where keys are `Oban` queues (see above) and values are numbers of max attempts for failed jobs. + ## Pleroma.Web.Metadata * `providers`: a list of metadata providers to enable. Providers available: * Pleroma.Web.Metadata.Providers.OpenGraph diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 844cd0732..a867ac998 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -1861,7 +1861,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do post(conn, "/api/pleroma/admin/config", %{ configs: [ %{ - "group" => "pleroma_job_queue", + "group" => "oban", "key" => ":queues", "value" => [ %{"tuple" => [":federator_incoming", 50]}, @@ -1879,7 +1879,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do assert json_response(conn, 200) == %{ "configs" => [ %{ - "group" => "pleroma_job_queue", + "group" => "oban", "key" => ":queues", "value" => [ %{"tuple" => [":federator_incoming", 50]}, From cd78e63a2528ab813088d5e44a026f6bb05b344b Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 27 Aug 2019 14:34:37 +0300 Subject: [PATCH 10/45] [#1149] Bugfix: Pleroma.Workers.Subscriber / "verify_websub" works with WebsubServerSubscription. --- lib/pleroma/workers/subscriber.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/workers/subscriber.ex b/lib/pleroma/workers/subscriber.ex index 783c44173..e960b35bf 100644 --- a/lib/pleroma/workers/subscriber.ex +++ b/lib/pleroma/workers/subscriber.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Workers.Subscriber do alias Pleroma.Repo alias Pleroma.Web.Federator - alias Pleroma.Web.Websub.WebsubClientSubscription + alias Pleroma.Web.Websub # Note: `max_attempts` is intended to be overridden in `new/1` call use Oban.Worker, @@ -18,12 +18,12 @@ defmodule Pleroma.Workers.Subscriber do end def perform(%{"op" => "request_subscription", "websub_id" => websub_id}, _job) do - websub = Repo.get(WebsubClientSubscription, websub_id) + websub = Repo.get(Websub.WebsubClientSubscription, websub_id) Federator.perform(:request_subscription, websub) end def perform(%{"op" => "verify_websub", "websub_id" => websub_id}, _job) do - websub = Repo.get(WebsubClientSubscription, websub_id) + websub = Repo.get(Websub.WebsubServerSubscription, websub_id) Federator.perform(:verify_websub, websub) end end From a90ea8ba1562818b025f677ffeea35f7ca08ddf2 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 31 Aug 2019 19:08:56 +0300 Subject: [PATCH 11/45] [#1149] Addressed code review comments (code style, jobs pruning etc.). --- CHANGELOG.md | 2 +- config/config.exs | 2 +- config/test.exs | 2 + docs/config.md | 56 ++++++++++++++++++- lib/pleroma/activity_expiration_worker.ex | 6 +- lib/pleroma/application.ex | 2 +- lib/pleroma/digest_email_worker.ex | 4 +- lib/pleroma/emails/mailer.ex | 4 +- lib/pleroma/scheduled_activity_worker.ex | 2 +- lib/pleroma/user.ex | 2 +- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- .../mrf/mediaproxy_warming_policy.ex | 2 +- lib/pleroma/web/activity_pub/publisher.ex | 2 +- .../web/activity_pub/transmogrifier.ex | 4 +- lib/pleroma/web/federator/federator.ex | 8 +-- lib/pleroma/web/federator/publisher.ex | 9 +-- lib/pleroma/web/oauth/token/clean_worker.ex | 2 +- lib/pleroma/web/push/push.ex | 6 +- lib/pleroma/web/salmon/salmon.ex | 2 +- .../workers/activity_expiration_worker.ex | 21 +++++++ lib/pleroma/workers/background_worker.ex | 19 ++----- lib/pleroma/workers/helper.ex | 13 ----- .../workers/{mailer.ex => mailer_worker.ex} | 19 +++---- .../{publisher.ex => publisher_worker.ex} | 8 ++- .../{receiver.ex => receiver_worker.ex} | 4 +- .../workers/scheduled_activity_worker.ex | 2 +- .../{subscriber.ex => subscriber_worker.ex} | 4 +- ...smogrifier.ex => transmogrifier_worker.ex} | 6 +- .../{web_pusher.ex => web_pusher_worker.ex} | 4 +- lib/pleroma/workers/worker_helper.ex | 23 ++++++++ test/user_test.exs | 2 +- .../activity_pub_controller_test.exs | 2 +- test/web/federator_test.exs | 2 +- test/web/websub/websub_test.exs | 2 +- 34 files changed, 163 insertions(+), 87 deletions(-) create mode 100644 lib/pleroma/workers/activity_expiration_worker.ex delete mode 100644 lib/pleroma/workers/helper.ex rename lib/pleroma/workers/{mailer.ex => mailer_worker.ex} (58%) rename lib/pleroma/workers/{publisher.ex => publisher_worker.ex} (76%) rename lib/pleroma/workers/{receiver.ex => receiver_worker.ex} (83%) rename lib/pleroma/workers/{subscriber.ex => subscriber_worker.ex} (88%) rename lib/pleroma/workers/{transmogrifier.ex => transmogrifier_worker.ex} (73%) rename lib/pleroma/workers/{web_pusher.ex => web_pusher_worker.ex} (82%) create mode 100644 lib/pleroma/workers/worker_helper.ex diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b73c783f..c9d6fef17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Unsubscribe followers when they unfollow a user - AdminAPI: Add "godmode" while fetching user statuses (i.e. admin can see private statuses) - Improve digest email template -- Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) with [Oban](https://github.com/sorentwo/oban) +- Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings). - Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler ### Fixed diff --git a/config/config.exs b/config/config.exs index da89aa3e9..6fb4a0969 100644 --- a/config/config.exs +++ b/config/config.exs @@ -470,7 +470,7 @@ config :pleroma, Pleroma.User, config :pleroma, Oban, repo: Pleroma.Repo, verbose: false, - prune: {:maxage, 60 * 60 * 24 * 7}, + prune: {:maxlen, 1500}, queues: [ activity_expiration: 10, federator_incoming: 50, diff --git a/config/test.exs b/config/test.exs index 0ef809ac1..df512b5d7 100644 --- a/config/test.exs +++ b/config/test.exs @@ -65,6 +65,8 @@ config :pleroma, Oban, queues: false, prune: :disabled +config :pleroma, Pleroma.Scheduler, jobs: [] + config :pleroma, Pleroma.ScheduledActivity, daily_user_limit: 2, total_user_limit: 3, diff --git a/docs/config.md b/docs/config.md index 2e351e272..29a4d4c97 100644 --- a/docs/config.md +++ b/docs/config.md @@ -404,20 +404,29 @@ curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerando [Oban](https://github.com/sorentwo/oban) asynchronous job processor configuration. +Configuration options described in [Oban readme](https://github.com/sorentwo/oban#usage): +* `repo` - app's Ecto repo (`Pleroma.Repo`) +* `verbose` - logs verbosity +* `prune` - non-retryable jobs [pruning settings](https://github.com/sorentwo/oban#pruning) (`:disabled` / `{:maxlen, value}` / `{:maxage, value}`) +* `queues` - job queues (see below) + Pleroma has the following queues: +* `activity_expiration` - Activity expiration * `federator_outgoing` - Outgoing federation * `federator_incoming` - Incoming federation -* `mailer` - Email sender, see [`Pleroma.Emails.Mailer`](#pleroma-emails-mailer) +* `mailer` - Email sender, see [`Pleroma.Emails.Mailer`](#pleromaemailsmailer) * `transmogrifier` - Transmogrifier * `web_push` - Web push notifications -* `scheduled_activities` - Scheduled activities, see [`Pleroma.ScheduledActivities`](#pleromascheduledactivity) +* `scheduled_activities` - Scheduled activities, see [`Pleroma.ScheduledActivity`](#pleromascheduledactivity) Example: ```elixir config :pleroma, Oban, repo: Pleroma.Repo, + verbose: false, + prune: {:maxlen, 1500}, queues: [ federator_incoming: 50, federator_outgoing: 50 @@ -426,12 +435,37 @@ config :pleroma, Oban, This config contains two queues: `federator_incoming` and `federator_outgoing`. Both have the number of max concurrent jobs set to `50`. +### Migrating `pleroma_job_queue` settings + +`config :pleroma_job_queue, :queues` is replaced by `config :pleroma, Oban, :queues` and uses the same format (keys are queues' names, values are max concurrent jobs numbers). + +### Note on running with PostgreSQL in silent mode + +If you are running PostgreSQL in [`silent_mode`](https://postgresqlco.nf/en/doc/param/silent_mode?version=9.1), it's advised to set [`log_destination`](https://postgresqlco.nf/en/doc/param/log_destination?version=9.1) to `syslog`, +otherwise `postmaster.log` file may grow because of "you don't own a lock of type ShareLock" warnings (see https://github.com/sorentwo/oban/issues/52). + ## :workers Includes custom worker options not interpretable directly by `Oban`. * `retries` — keyword lists where keys are `Oban` queues (see above) and values are numbers of max attempts for failed jobs. +Example: + +```elixir +config :pleroma, :workers, + retries: [ + federator_incoming: 5, + federator_outgoing: 5 + ] +``` + +### Migrating `Pleroma.Web.Federator.RetryQueue` settings + +* `max_retries` is replaced with `config :pleroma, :workers, retries: [federator_outgoing: 5]` +* `enabled: false` corresponds to `config :pleroma, :workers, retries: [federator_outgoing: 1]` +* deprecated options: `max_jobs`, `initial_timeout` + ## Pleroma.Web.Metadata * `providers`: a list of metadata providers to enable. Providers available: * Pleroma.Web.Metadata.Providers.OpenGraph @@ -491,6 +525,24 @@ config :auto_linker, ] ``` +## Pleroma.Scheduler + +Configuration for [Quantum](https://github.com/quantum-elixir/quantum-core) jobs scheduler. + +See [Quantum readme](https://github.com/quantum-elixir/quantum-core#usage) for the list of supported options. + +Example: + +```elixir +config :pleroma, Pleroma.Scheduler, + global: true, + overlap: true, + timezone: :utc, + jobs: [{"0 */6 * * * *", {Pleroma.Web.Websub, :refresh_subscriptions, []}}] +``` + +The above example defines a single job which invokes `Pleroma.Web.Websub.refresh_subscriptions()` every 6 hours ("0 */6 * * * *", [crontab format](https://en.wikipedia.org/wiki/Cron)). + ## Pleroma.ScheduledActivity * `daily_user_limit`: the number of scheduled activities a user is allowed to create in a single day (Default: `25`) diff --git a/lib/pleroma/activity_expiration_worker.ex b/lib/pleroma/activity_expiration_worker.ex index 5c0c53232..7aba7eece 100644 --- a/lib/pleroma/activity_expiration_worker.ex +++ b/lib/pleroma/activity_expiration_worker.ex @@ -9,13 +9,13 @@ defmodule Pleroma.ActivityExpirationWorker do alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.CommonAPI - alias Pleroma.Workers.BackgroundWorker + alias Pleroma.Workers.ActivityExpirationWorker require Logger use GenServer import Ecto.Query - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] @schedule_interval :timer.minutes(1) @@ -57,7 +57,7 @@ defmodule Pleroma.ActivityExpirationWorker do "op" => "activity_expiration", "activity_expiration_id" => expiration.id } - |> BackgroundWorker.new(worker_args(:activity_expiration)) + |> ActivityExpirationWorker.new(worker_args(:activity_expiration)) |> Repo.insert() end) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 7d38ed5c4..f8f866dbd 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -43,7 +43,7 @@ defmodule Pleroma.Application do hackney_pool_children() ++ [ Pleroma.Stats, - {Oban, Application.get_env(:pleroma, Oban)}, + {Oban, Pleroma.Config.get(Oban)}, %{ id: :web_push_init, start: {Task, :start_link, [&Pleroma.Web.Push.init/0]}, diff --git a/lib/pleroma/digest_email_worker.ex b/lib/pleroma/digest_email_worker.ex index ffc48bfab..4ab2a4ef4 100644 --- a/lib/pleroma/digest_email_worker.ex +++ b/lib/pleroma/digest_email_worker.ex @@ -4,11 +4,11 @@ defmodule Pleroma.DigestEmailWorker do alias Pleroma.Repo - alias Pleroma.Workers.Mailer, as: MailerWorker + alias Pleroma.Workers.MailerWorker import Ecto.Query - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] def perform do config = Pleroma.Config.get([:email_notifications, :digest]) diff --git a/lib/pleroma/emails/mailer.ex b/lib/pleroma/emails/mailer.ex index bb534f602..9cbe7313c 100644 --- a/lib/pleroma/emails/mailer.ex +++ b/lib/pleroma/emails/mailer.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Emails.Mailer do """ alias Pleroma.Repo - alias Pleroma.Workers.Mailer, as: MailerWorker + alias Pleroma.Workers.MailerWorker alias Swoosh.DeliveryError @otp_app :pleroma @@ -19,7 +19,7 @@ defmodule Pleroma.Emails.Mailer do @spec enabled?() :: boolean() def enabled?, do: Pleroma.Config.get([__MODULE__, :enabled]) - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] @doc "add email to queue" def deliver_async(email, config \\ []) do diff --git a/lib/pleroma/scheduled_activity_worker.ex b/lib/pleroma/scheduled_activity_worker.ex index a01fb4fcb..8bf534f42 100644 --- a/lib/pleroma/scheduled_activity_worker.ex +++ b/lib/pleroma/scheduled_activity_worker.ex @@ -18,7 +18,7 @@ defmodule Pleroma.ScheduledActivityWorker do @schedule_interval :timer.minutes(1) - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] def start_link(_) do GenServer.start_link(__MODULE__, nil) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 18bba0fbb..abfa063fb 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -41,7 +41,7 @@ defmodule Pleroma.User do @strict_local_nickname_regex ~r/^[a-zA-Z\d]+$/ @extended_local_nickname_regex ~r/^[a-zA-Z\d_-]+$/ - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] schema "users" do field(:bio, :string) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 50279cca5..74c5eb91c 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -26,7 +26,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do require Logger require Pleroma.Constants - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] # For Announce activities, we filter the recipients based on following status for any actors # that match actual users. See issue #164 for more information about why this is necessary. diff --git a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex index b188164ee..178321558 100644 --- a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex @@ -18,7 +18,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do recv_timeout: 10_000 ] - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] def perform(:prefetch, url) do Logger.info("Prefetching #{inspect(url)}") diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 24d101dc8..a6322e25a 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -85,7 +85,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do end def publish_one(%{actor_id: actor_id} = params) do - actor = User.get_by_id(actor_id) + actor = User.get_cached_by_id(actor_id) params |> Map.delete(:actor_id) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index b068d28a7..9437f9a16 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -15,14 +15,14 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.Federator - alias Pleroma.Workers.Transmogrifier, as: TransmogrifierWorker + alias Pleroma.Workers.TransmogrifierWorker import Ecto.Query require Logger require Pleroma.Constants - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] @doc """ Modifies an incoming AP object (mastodon format) to our internal format. diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index cf7e50fee..8f43066e3 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -12,13 +12,13 @@ defmodule Pleroma.Web.Federator do alias Pleroma.Web.Federator.Publisher alias Pleroma.Web.OStatus alias Pleroma.Web.Websub - alias Pleroma.Workers.Publisher, as: PublisherWorker - alias Pleroma.Workers.Receiver, as: ReceiverWorker - alias Pleroma.Workers.Subscriber, as: SubscriberWorker + alias Pleroma.Workers.PublisherWorker + alias Pleroma.Workers.ReceiverWorker + alias Pleroma.Workers.SubscriberWorker require Logger - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] def init do # To do: consider removing this call in favor of scheduled execution (`quantum`-based) diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex index 05d2be615..42be109ab 100644 --- a/lib/pleroma/web/federator/publisher.ex +++ b/lib/pleroma/web/federator/publisher.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.Federator.Publisher do alias Pleroma.Activity alias Pleroma.Config alias Pleroma.User - alias Pleroma.Workers.Publisher, as: PublisherWorker + alias Pleroma.Workers.PublisherWorker require Logger @@ -31,12 +31,7 @@ defmodule Pleroma.Web.Federator.Publisher do """ @spec enqueue_one(module(), Map.t()) :: :ok def enqueue_one(module, %{} = params) do - worker_args = - if max_attempts = Pleroma.Config.get([:workers, :retries, :federator_outgoing]) do - [max_attempts: max_attempts] - else - [] - end + worker_args = Pleroma.Workers.WorkerHelper.worker_args(:federator_outgoing) %{"op" => "publish_one", "module" => to_string(module), "params" => params} |> PublisherWorker.new(worker_args) diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex index 943e73289..b150a68a7 100644 --- a/lib/pleroma/web/oauth/token/clean_worker.ex +++ b/lib/pleroma/web/oauth/token/clean_worker.ex @@ -20,7 +20,7 @@ defmodule Pleroma.Web.OAuth.Token.CleanWorker do alias Pleroma.Web.OAuth.Token alias Pleroma.Workers.BackgroundWorker - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] def start_link(_), do: GenServer.start_link(__MODULE__, %{}) diff --git a/lib/pleroma/web/push/push.ex b/lib/pleroma/web/push/push.ex index b4f0e5127..4973b529c 100644 --- a/lib/pleroma/web/push/push.ex +++ b/lib/pleroma/web/push/push.ex @@ -4,11 +4,11 @@ defmodule Pleroma.Web.Push do alias Pleroma.Repo - alias Pleroma.Workers.WebPusher + alias Pleroma.Workers.WebPusherWorker require Logger - defdelegate worker_args(queue), to: Pleroma.Workers.Helper + import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] def init do unless enabled() do @@ -36,7 +36,7 @@ defmodule Pleroma.Web.Push do def send(notification) do %{"op" => "web_push", "notification_id" => notification.id} - |> WebPusher.new(worker_args(:web_push)) + |> WebPusherWorker.new(worker_args(:web_push)) |> Repo.insert() end end diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index bbaa293fd..8ba7380c0 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -171,7 +171,7 @@ defmodule Pleroma.Web.Salmon do end def publish_one(%{recipient_id: recipient_id} = params) do - recipient = User.get_by_id(recipient_id) + recipient = User.get_cached_by_id(recipient_id) params |> Map.delete(:recipient_id) diff --git a/lib/pleroma/workers/activity_expiration_worker.ex b/lib/pleroma/workers/activity_expiration_worker.ex new file mode 100644 index 000000000..0b491eabb --- /dev/null +++ b/lib/pleroma/workers/activity_expiration_worker.ex @@ -0,0 +1,21 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.ActivityExpirationWorker do + # Note: `max_attempts` is intended to be overridden in `new/2` call + use Oban.Worker, + queue: "activity_expiration", + max_attempts: 1 + + @impl Oban.Worker + def perform( + %{ + "op" => "activity_expiration", + "activity_expiration_id" => activity_expiration_id + }, + _job + ) do + Pleroma.ActivityExpirationWorker.perform(:execute, activity_expiration_id) + end +end diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index fbce7d789..7b5575a5f 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -8,24 +8,24 @@ defmodule Pleroma.Workers.BackgroundWorker do alias Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy alias Pleroma.Web.OAuth.Token.CleanWorker - # Note: `max_attempts` is intended to be overridden in `new/1` call + # Note: `max_attempts` is intended to be overridden in `new/2` call use Oban.Worker, queue: "background", max_attempts: 1 @impl Oban.Worker def perform(%{"op" => "fetch_initial_posts", "user_id" => user_id}, _job) do - user = User.get_by_id(user_id) + user = User.get_cached_by_id(user_id) User.perform(:fetch_initial_posts, user) end def perform(%{"op" => "deactivate_user", "user_id" => user_id, "status" => status}, _job) do - user = User.get_by_id(user_id) + user = User.get_cached_by_id(user_id) User.perform(:deactivate_async, user, status) end def perform(%{"op" => "delete_user", "user_id" => user_id}, _job) do - user = User.get_by_id(user_id) + user = User.get_cached_by_id(user_id) User.perform(:delete, user) end @@ -37,7 +37,7 @@ defmodule Pleroma.Workers.BackgroundWorker do }, _job ) do - blocker = User.get_by_id(blocker_id) + blocker = User.get_cached_by_id(blocker_id) User.perform(:blocks_import, blocker, blocked_identifiers) end @@ -49,7 +49,7 @@ defmodule Pleroma.Workers.BackgroundWorker do }, _job ) do - follower = User.get_by_id(follower_id) + follower = User.get_cached_by_id(follower_id) User.perform(:follow_import, follower, followed_identifiers) end @@ -69,11 +69,4 @@ defmodule Pleroma.Workers.BackgroundWorker do activity = Activity.get_by_id(activity_id) Pleroma.Web.RichMedia.Helpers.perform(:fetch, activity) end - - def perform( - %{"op" => "activity_expiration", "activity_expiration_id" => activity_expiration_id}, - _job - ) do - Pleroma.ActivityExpirationWorker.perform(:execute, activity_expiration_id) - end end diff --git a/lib/pleroma/workers/helper.ex b/lib/pleroma/workers/helper.ex deleted file mode 100644 index 3286ce0e8..000000000 --- a/lib/pleroma/workers/helper.ex +++ /dev/null @@ -1,13 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.Helper do - def worker_args(queue) do - if max_attempts = Pleroma.Config.get([:workers, :retries, queue]) do - [max_attempts: max_attempts] - else - [] - end - end -end diff --git a/lib/pleroma/workers/mailer.ex b/lib/pleroma/workers/mailer_worker.ex similarity index 58% rename from lib/pleroma/workers/mailer.ex rename to lib/pleroma/workers/mailer_worker.ex index 1cce2ea03..4f73d61bc 100644 --- a/lib/pleroma/workers/mailer.ex +++ b/lib/pleroma/workers/mailer_worker.ex @@ -2,26 +2,25 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Workers.Mailer do +defmodule Pleroma.Workers.MailerWorker do alias Pleroma.User - # Note: `max_attempts` is intended to be overridden in `new/1` call + # Note: `max_attempts` is intended to be overridden in `new/2` call use Oban.Worker, queue: "mailer", max_attempts: 1 @impl Oban.Worker def perform(%{"op" => "email", "encoded_email" => encoded_email, "config" => config}, _job) do - email = - encoded_email - |> Base.decode64!() - |> :erlang.binary_to_term() - - Pleroma.Emails.Mailer.deliver(email, config) + encoded_email + |> Base.decode64!() + |> :erlang.binary_to_term() + |> Pleroma.Emails.Mailer.deliver(config) end def perform(%{"op" => "digest_email", "user_id" => user_id}, _job) do - user = User.get_by_id(user_id) - Pleroma.DigestEmailWorker.perform(user) + user_id + |> User.get_cached_by_id() + |> Pleroma.DigestEmailWorker.perform() end end diff --git a/lib/pleroma/workers/publisher.ex b/lib/pleroma/workers/publisher_worker.ex similarity index 76% rename from lib/pleroma/workers/publisher.ex rename to lib/pleroma/workers/publisher_worker.ex index 00fae99c7..5671d2a29 100644 --- a/lib/pleroma/workers/publisher.ex +++ b/lib/pleroma/workers/publisher_worker.ex @@ -2,15 +2,19 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Workers.Publisher do +defmodule Pleroma.Workers.PublisherWorker do alias Pleroma.Activity alias Pleroma.Web.Federator - # Note: `max_attempts` is intended to be overridden in `new/1` call + # Note: `max_attempts` is intended to be overridden in `new/2` call use Oban.Worker, queue: "federator_outgoing", max_attempts: 1 + def backoff(attempt) when is_integer(attempt) do + Pleroma.Workers.WorkerHelper.sidekiq_backoff(attempt, 5) + end + @impl Oban.Worker def perform(%{"op" => "publish", "activity_id" => activity_id}, _job) do activity = Activity.get_by_id(activity_id) diff --git a/lib/pleroma/workers/receiver.ex b/lib/pleroma/workers/receiver_worker.ex similarity index 83% rename from lib/pleroma/workers/receiver.ex rename to lib/pleroma/workers/receiver_worker.ex index 4ee270d74..cdce630f2 100644 --- a/lib/pleroma/workers/receiver.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -2,10 +2,10 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Workers.Receiver do +defmodule Pleroma.Workers.ReceiverWorker do alias Pleroma.Web.Federator - # Note: `max_attempts` is intended to be overridden in `new/1` call + # Note: `max_attempts` is intended to be overridden in `new/2` call use Oban.Worker, queue: "federator_incoming", max_attempts: 1 diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex index d9724c78a..4094411ae 100644 --- a/lib/pleroma/workers/scheduled_activity_worker.ex +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.ScheduledActivityWorker do - # Note: `max_attempts` is intended to be overridden in `new/1` call + # Note: `max_attempts` is intended to be overridden in `new/2` call use Oban.Worker, queue: "scheduled_activities", max_attempts: 1 diff --git a/lib/pleroma/workers/subscriber.ex b/lib/pleroma/workers/subscriber_worker.ex similarity index 88% rename from lib/pleroma/workers/subscriber.ex rename to lib/pleroma/workers/subscriber_worker.ex index e960b35bf..22d1dc956 100644 --- a/lib/pleroma/workers/subscriber.ex +++ b/lib/pleroma/workers/subscriber_worker.ex @@ -2,12 +2,12 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Workers.Subscriber do +defmodule Pleroma.Workers.SubscriberWorker do alias Pleroma.Repo alias Pleroma.Web.Federator alias Pleroma.Web.Websub - # Note: `max_attempts` is intended to be overridden in `new/1` call + # Note: `max_attempts` is intended to be overridden in `new/2` call use Oban.Worker, queue: "federator_outgoing", max_attempts: 1 diff --git a/lib/pleroma/workers/transmogrifier.ex b/lib/pleroma/workers/transmogrifier_worker.ex similarity index 73% rename from lib/pleroma/workers/transmogrifier.ex rename to lib/pleroma/workers/transmogrifier_worker.ex index e13202c06..6f5c1a2f2 100644 --- a/lib/pleroma/workers/transmogrifier.ex +++ b/lib/pleroma/workers/transmogrifier_worker.ex @@ -2,17 +2,17 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Workers.Transmogrifier do +defmodule Pleroma.Workers.TransmogrifierWorker do alias Pleroma.User - # Note: `max_attempts` is intended to be overridden in `new/1` call + # Note: `max_attempts` is intended to be overridden in `new/2` call use Oban.Worker, queue: "transmogrifier", max_attempts: 1 @impl Oban.Worker def perform(%{"op" => "user_upgrade", "user_id" => user_id}, _job) do - user = User.get_by_id(user_id) + user = User.get_cached_by_id(user_id) Pleroma.Web.ActivityPub.Transmogrifier.perform(:user_upgrade, user) end end diff --git a/lib/pleroma/workers/web_pusher.ex b/lib/pleroma/workers/web_pusher_worker.ex similarity index 82% rename from lib/pleroma/workers/web_pusher.ex rename to lib/pleroma/workers/web_pusher_worker.ex index 7b78bb3ea..2b1d3b99a 100644 --- a/lib/pleroma/workers/web_pusher.ex +++ b/lib/pleroma/workers/web_pusher_worker.ex @@ -2,11 +2,11 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Workers.WebPusher do +defmodule Pleroma.Workers.WebPusherWorker do alias Pleroma.Notification alias Pleroma.Repo - # Note: `max_attempts` is intended to be overridden in `new/1` call + # Note: `max_attempts` is intended to be overridden in `new/2` call use Oban.Worker, queue: "web_push", max_attempts: 1 diff --git a/lib/pleroma/workers/worker_helper.ex b/lib/pleroma/workers/worker_helper.ex new file mode 100644 index 000000000..f9ed2e64d --- /dev/null +++ b/lib/pleroma/workers/worker_helper.ex @@ -0,0 +1,23 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.WorkerHelper do + alias Pleroma.Config + + def worker_args(queue) do + case Config.get([:workers, :retries, queue]) do + nil -> [] + max_attempts -> [max_attempts: max_attempts] + end + end + + def sidekiq_backoff(attempt, pow \\ 4, base_backoff \\ 15) do + backoff = + :math.pow(attempt, pow) + + base_backoff + + :rand.uniform(2 * base_backoff) * attempt + + trunc(backoff) + end +end diff --git a/test/user_test.exs b/test/user_test.exs index 86232de99..0acd0db4e 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1123,7 +1123,7 @@ defmodule Pleroma.UserTest do "id" => "pleroma:fakeid" } }, - all_enqueued(worker: Pleroma.Workers.Publisher) + all_enqueued(worker: Pleroma.Workers.PublisherWorker) ) end end diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index a1b567a46..f1c1bb503 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -17,7 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do alias Pleroma.Web.ActivityPub.UserView alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.CommonAPI - alias Pleroma.Workers.Receiver, as: ReceiverWorker + alias Pleroma.Workers.ReceiverWorker setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index 5724672fd..4096d4690 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -7,7 +7,7 @@ defmodule Pleroma.Web.FederatorTest do alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.CommonAPI alias Pleroma.Web.Federator - alias Pleroma.Workers.Publisher, as: PublisherWorker + alias Pleroma.Workers.PublisherWorker use Pleroma.DataCase use Oban.Testing, repo: Pleroma.Repo diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index 414610879..929acf5a2 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -11,7 +11,7 @@ defmodule Pleroma.Web.WebsubTest do alias Pleroma.Web.Websub alias Pleroma.Web.Websub.WebsubClientSubscription alias Pleroma.Web.Websub.WebsubServerSubscription - alias Pleroma.Workers.Subscriber, as: SubscriberWorker + alias Pleroma.Workers.SubscriberWorker import Pleroma.Factory import Tesla.Mock From dd017c65a4b86501c435f5cb01804300e6b7c6dd Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 31 Aug 2019 21:58:42 +0300 Subject: [PATCH 12/45] [#1149] Refactored Oban workers API (introduced `enqueue/3`). --- lib/pleroma/activity_expiration_worker.ex | 13 +++------ lib/pleroma/digest_email_worker.ex | 10 ++----- lib/pleroma/emails/mailer.ex | 7 +---- lib/pleroma/scheduled_activity_worker.ex | 10 +++---- lib/pleroma/user.ex | 28 +++++-------------- lib/pleroma/web/activity_pub/activity_pub.ex | 6 +--- .../mrf/mediaproxy_warming_policy.ex | 11 ++------ .../web/activity_pub/transmogrifier.ex | 6 +--- lib/pleroma/web/federator/federator.ex | 26 ++++------------- lib/pleroma/web/federator/publisher.ex | 9 +++--- lib/pleroma/web/oauth/token/clean_worker.ex | 7 +---- lib/pleroma/web/push/push.ex | 7 +---- .../workers/activity_expiration_worker.ex | 2 ++ lib/pleroma/workers/background_worker.ex | 2 ++ lib/pleroma/workers/digest_emails_worker.ex | 21 ++++++++++++++ lib/pleroma/workers/mailer_worker.ex | 10 ++----- lib/pleroma/workers/publisher_worker.ex | 2 ++ lib/pleroma/workers/receiver_worker.ex | 2 ++ .../workers/scheduled_activity_worker.ex | 2 ++ lib/pleroma/workers/subscriber_worker.ex | 2 ++ lib/pleroma/workers/transmogrifier_worker.ex | 2 ++ lib/pleroma/workers/web_pusher_worker.ex | 2 ++ lib/pleroma/workers/worker_helper.ex | 18 ++++++++++++ 23 files changed, 92 insertions(+), 113 deletions(-) create mode 100644 lib/pleroma/workers/digest_emails_worker.ex diff --git a/lib/pleroma/activity_expiration_worker.ex b/lib/pleroma/activity_expiration_worker.ex index 7aba7eece..c0820c202 100644 --- a/lib/pleroma/activity_expiration_worker.ex +++ b/lib/pleroma/activity_expiration_worker.ex @@ -9,14 +9,11 @@ defmodule Pleroma.ActivityExpirationWorker do alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.CommonAPI - alias Pleroma.Workers.ActivityExpirationWorker require Logger use GenServer import Ecto.Query - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - @schedule_interval :timer.minutes(1) def start_link(_) do @@ -53,12 +50,10 @@ defmodule Pleroma.ActivityExpirationWorker do def handle_info(:perform, state) do ActivityExpiration.due_expirations(@schedule_interval) |> Enum.each(fn expiration -> - %{ - "op" => "activity_expiration", - "activity_expiration_id" => expiration.id - } - |> ActivityExpirationWorker.new(worker_args(:activity_expiration)) - |> Repo.insert() + Pleroma.Workers.ActivityExpirationWorker.enqueue( + "activity_expiration", + %{"activity_expiration_id" => expiration.id} + ) end) schedule_next() diff --git a/lib/pleroma/digest_email_worker.ex b/lib/pleroma/digest_email_worker.ex index 4ab2a4ef4..5be7cf26b 100644 --- a/lib/pleroma/digest_email_worker.ex +++ b/lib/pleroma/digest_email_worker.ex @@ -4,12 +4,10 @@ defmodule Pleroma.DigestEmailWorker do alias Pleroma.Repo - alias Pleroma.Workers.MailerWorker + alias Pleroma.Workers.DigestEmailsWorker import Ecto.Query - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - def perform do config = Pleroma.Config.get([:email_notifications, :digest]) negative_interval = -Map.fetch!(config, :interval) @@ -23,11 +21,9 @@ defmodule Pleroma.DigestEmailWorker do where: u.last_digest_emailed_at < datetime_add(^now, ^negative_interval, "day"), select: u ) - |> Pleroma.Repo.all() + |> Repo.all() |> Enum.each(fn user -> - %{"op" => "digest_email", "user_id" => user.id} - |> MailerWorker.new([queue: "digest_emails"] ++ worker_args(:digest_emails)) - |> Repo.insert() + DigestEmailsWorker.enqueue("digest_email", %{"user_id" => user.id}) end) end diff --git a/lib/pleroma/emails/mailer.ex b/lib/pleroma/emails/mailer.ex index 9cbe7313c..eb96f2e8b 100644 --- a/lib/pleroma/emails/mailer.ex +++ b/lib/pleroma/emails/mailer.ex @@ -9,7 +9,6 @@ defmodule Pleroma.Emails.Mailer do The module contains functions to delivery email using Swoosh.Mailer. """ - alias Pleroma.Repo alias Pleroma.Workers.MailerWorker alias Swoosh.DeliveryError @@ -19,8 +18,6 @@ defmodule Pleroma.Emails.Mailer do @spec enabled?() :: boolean() def enabled?, do: Pleroma.Config.get([__MODULE__, :enabled]) - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - @doc "add email to queue" def deliver_async(email, config \\ []) do encoded_email = @@ -28,9 +25,7 @@ defmodule Pleroma.Emails.Mailer do |> :erlang.term_to_binary() |> Base.encode64() - %{"op" => "email", "encoded_email" => encoded_email, "config" => config} - |> MailerWorker.new(worker_args(:mailer)) - |> Repo.insert() + MailerWorker.enqueue("email", %{"encoded_email" => encoded_email, "config" => config}) end @doc "callback to perform send email from queue" diff --git a/lib/pleroma/scheduled_activity_worker.ex b/lib/pleroma/scheduled_activity_worker.ex index 8bf534f42..c41a542de 100644 --- a/lib/pleroma/scheduled_activity_worker.ex +++ b/lib/pleroma/scheduled_activity_worker.ex @@ -8,7 +8,6 @@ defmodule Pleroma.ScheduledActivityWorker do """ alias Pleroma.Config - alias Pleroma.Repo alias Pleroma.ScheduledActivity alias Pleroma.User alias Pleroma.Web.CommonAPI @@ -18,8 +17,6 @@ defmodule Pleroma.ScheduledActivityWorker do @schedule_interval :timer.minutes(1) - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - def start_link(_) do GenServer.start_link(__MODULE__, nil) end @@ -49,9 +46,10 @@ defmodule Pleroma.ScheduledActivityWorker do def handle_info(:perform, state) do ScheduledActivity.due_activities(@schedule_interval) |> Enum.each(fn scheduled_activity -> - %{"op" => "execute", "activity_id" => scheduled_activity.id} - |> Pleroma.Workers.ScheduledActivityWorker.new(worker_args(:scheduled_activities)) - |> Repo.insert() + Pleroma.Workers.ScheduledActivityWorker.enqueue( + "execute", + %{"activity_id" => scheduled_activity.id} + ) end) schedule_next() diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index abfa063fb..2fe7e1748 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -41,8 +41,6 @@ defmodule Pleroma.User do @strict_local_nickname_regex ~r/^[a-zA-Z\d]+$/ @extended_local_nickname_regex ~r/^[a-zA-Z\d_-]+$/ - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - schema "users" do field(:bio, :string) field(:email, :string) @@ -623,9 +621,7 @@ defmodule Pleroma.User do @doc "Fetch some posts when the user has just been federated with" def fetch_initial_posts(user) do - %{"op" => "fetch_initial_posts", "user_id" => user.id} - |> BackgroundWorker.new(worker_args(:background)) - |> Repo.insert() + BackgroundWorker.enqueue("fetch_initial_posts", %{"user_id" => user.id}) end @spec get_followers_query(User.t(), pos_integer() | nil) :: Ecto.Query.t() @@ -1056,9 +1052,7 @@ defmodule Pleroma.User do end def deactivate_async(user, status \\ true) do - %{"op" => "deactivate_user", "user_id" => user.id, "status" => status} - |> BackgroundWorker.new(worker_args(:background)) - |> Repo.insert() + BackgroundWorker.enqueue("deactivate_user", %{"user_id" => user.id, "status" => status}) end def deactivate(%User{} = user, status \\ true) do @@ -1087,9 +1081,7 @@ defmodule Pleroma.User do end def delete(%User{} = user) do - %{"op" => "delete_user", "user_id" => user.id} - |> BackgroundWorker.new(worker_args(:background)) - |> Repo.insert() + BackgroundWorker.enqueue("delete_user", %{"user_id" => user.id}) end @spec perform(atom(), User.t()) :: {:ok, User.t()} @@ -1198,24 +1190,18 @@ defmodule Pleroma.User do end def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers) do - %{ - "op" => "blocks_import", + BackgroundWorker.enqueue("blocks_import", %{ "blocker_id" => blocker.id, "blocked_identifiers" => blocked_identifiers - } - |> BackgroundWorker.new(worker_args(:background)) - |> Repo.insert() + }) end def follow_import(%User{} = follower, followed_identifiers) when is_list(followed_identifiers) do - %{ - "op" => "follow_import", + BackgroundWorker.enqueue("follow_import", %{ "follower_id" => follower.id, "followed_identifiers" => followed_identifiers - } - |> BackgroundWorker.new(worker_args(:background)) - |> Repo.insert() + }) end def delete_user_activities(%User{ap_id: ap_id} = user) do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 74c5eb91c..90b409606 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -26,8 +26,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do require Logger require Pleroma.Constants - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - # For Announce activities, we filter the recipients based on following status for any actors # that match actual users. See issue #164 for more information about why this is necessary. defp get_recipients(%{"type" => "Announce"} = data) do @@ -148,9 +146,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do activity end - %{"op" => "fetch_data_for_activity", "activity_id" => activity.id} - |> BackgroundWorker.new(worker_args(:background)) - |> Repo.insert() + BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id}) Notification.create_notifications(activity) diff --git a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex index 178321558..26b8539fe 100644 --- a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex @@ -7,7 +7,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do @behaviour Pleroma.Web.ActivityPub.MRF alias Pleroma.HTTP - alias Pleroma.Repo alias Pleroma.Web.MediaProxy alias Pleroma.Workers.BackgroundWorker @@ -18,8 +17,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do recv_timeout: 10_000 ] - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - def perform(:prefetch, url) do Logger.info("Prefetching #{inspect(url)}") @@ -34,9 +31,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do url |> Enum.each(fn %{"href" => href} -> - %{"op" => "media_proxy_prefetch", "url" => href} - |> BackgroundWorker.new(worker_args(:background)) - |> Repo.insert() + BackgroundWorker.enqueue("media_proxy_prefetch", %{"url" => href}) x -> Logger.debug("Unhandled attachment URL object #{inspect(x)}") @@ -52,9 +47,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do %{"type" => "Create", "object" => %{"attachment" => attachments} = _object} = message ) when is_list(attachments) and length(attachments) > 0 do - %{"op" => "media_proxy_preload", "message" => message} - |> BackgroundWorker.new(worker_args(:background)) - |> Repo.insert() + BackgroundWorker.enqueue("media_proxy_preload", %{"message" => message}) {:ok, message} end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 9437f9a16..f27455e8b 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -22,8 +22,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do require Logger require Pleroma.Constants - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - @doc """ Modifies an incoming AP object (mastodon format) to our internal format. """ @@ -1054,9 +1052,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do already_ap <- User.ap_enabled?(user), {:ok, user} <- user |> User.upgrade_changeset(data) |> User.update_and_set_cache() do unless already_ap do - %{"op" => "user_upgrade", "user_id" => user.id} - |> TransmogrifierWorker.new(worker_args(:transmogrifier)) - |> Repo.insert() + TransmogrifierWorker.enqueue("user_upgrade", %{"user_id" => user.id}) end {:ok, user} diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 8f43066e3..1a2da014a 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -18,8 +18,6 @@ defmodule Pleroma.Web.Federator do require Logger - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - def init do # To do: consider removing this call in favor of scheduled execution (`quantum`-based) refresh_subscriptions(schedule_in: 60) @@ -40,15 +38,11 @@ defmodule Pleroma.Web.Federator do # Client API def incoming_doc(doc) do - %{"op" => "incoming_doc", "body" => doc} - |> ReceiverWorker.new(worker_args(:federator_incoming)) - |> Pleroma.Repo.insert() + ReceiverWorker.enqueue("incoming_doc", %{"body" => doc}) end def incoming_ap_doc(params) do - %{"op" => "incoming_ap_doc", "params" => params} - |> ReceiverWorker.new(worker_args(:federator_incoming)) - |> Pleroma.Repo.insert() + ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}) end def publish(%{id: "pleroma:fakeid"} = activity) do @@ -56,27 +50,19 @@ defmodule Pleroma.Web.Federator do end def publish(activity) do - %{"op" => "publish", "activity_id" => activity.id} - |> PublisherWorker.new(worker_args(:federator_outgoing)) - |> Pleroma.Repo.insert() + PublisherWorker.enqueue("publish", %{"activity_id" => activity.id}) end def verify_websub(websub) do - %{"op" => "verify_websub", "websub_id" => websub.id} - |> SubscriberWorker.new(worker_args(:federator_outgoing)) - |> Pleroma.Repo.insert() + SubscriberWorker.enqueue("verify_websub", %{"websub_id" => websub.id}) end def request_subscription(websub) do - %{"op" => "request_subscription", "websub_id" => websub.id} - |> SubscriberWorker.new(worker_args(:federator_outgoing)) - |> Pleroma.Repo.insert() + SubscriberWorker.enqueue("request_subscription", %{"websub_id" => websub.id}) end def refresh_subscriptions(worker_args \\ []) do - %{"op" => "refresh_subscriptions"} - |> SubscriberWorker.new(worker_args ++ [max_attempts: 1] ++ worker_args(:federator_outgoing)) - |> Pleroma.Repo.insert() + SubscriberWorker.enqueue("refresh_subscriptions", %{}, worker_args ++ [max_attempts: 1]) end # Job Worker Callbacks diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex index 42be109ab..937064638 100644 --- a/lib/pleroma/web/federator/publisher.ex +++ b/lib/pleroma/web/federator/publisher.ex @@ -31,11 +31,10 @@ defmodule Pleroma.Web.Federator.Publisher do """ @spec enqueue_one(module(), Map.t()) :: :ok def enqueue_one(module, %{} = params) do - worker_args = Pleroma.Workers.WorkerHelper.worker_args(:federator_outgoing) - - %{"op" => "publish_one", "module" => to_string(module), "params" => params} - |> PublisherWorker.new(worker_args) - |> Pleroma.Repo.insert() + PublisherWorker.enqueue( + "publish_one", + %{"module" => to_string(module), "params" => params} + ) end @doc """ diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex index b150a68a7..eb94bf86f 100644 --- a/lib/pleroma/web/oauth/token/clean_worker.ex +++ b/lib/pleroma/web/oauth/token/clean_worker.ex @@ -16,12 +16,9 @@ defmodule Pleroma.Web.OAuth.Token.CleanWorker do @one_day ) - alias Pleroma.Repo alias Pleroma.Web.OAuth.Token alias Pleroma.Workers.BackgroundWorker - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - def start_link(_), do: GenServer.start_link(__MODULE__, %{}) def init(_) do @@ -31,9 +28,7 @@ defmodule Pleroma.Web.OAuth.Token.CleanWorker do @doc false def handle_info(:perform, state) do - %{"op" => "clean_expired_tokens"} - |> BackgroundWorker.new(worker_args(:background)) - |> Repo.insert() + BackgroundWorker.enqueue("clean_expired_tokens", %{}) Process.send_after(self(), :perform, @interval) {:noreply, state} diff --git a/lib/pleroma/web/push/push.ex b/lib/pleroma/web/push/push.ex index 4973b529c..7ef1532ac 100644 --- a/lib/pleroma/web/push/push.ex +++ b/lib/pleroma/web/push/push.ex @@ -3,13 +3,10 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Push do - alias Pleroma.Repo alias Pleroma.Workers.WebPusherWorker require Logger - import Pleroma.Workers.WorkerHelper, only: [worker_args: 1] - def init do unless enabled() do Logger.warn(""" @@ -35,8 +32,6 @@ defmodule Pleroma.Web.Push do end def send(notification) do - %{"op" => "web_push", "notification_id" => notification.id} - |> WebPusherWorker.new(worker_args(:web_push)) - |> Repo.insert() + WebPusherWorker.enqueue("web_push", %{"notification_id" => notification.id}) end end diff --git a/lib/pleroma/workers/activity_expiration_worker.ex b/lib/pleroma/workers/activity_expiration_worker.ex index 0b491eabb..60dd3feba 100644 --- a/lib/pleroma/workers/activity_expiration_worker.ex +++ b/lib/pleroma/workers/activity_expiration_worker.ex @@ -8,6 +8,8 @@ defmodule Pleroma.Workers.ActivityExpirationWorker do queue: "activity_expiration", max_attempts: 1 + use Pleroma.Workers.WorkerHelper, queue: "activity_expiration" + @impl Oban.Worker def perform( %{ diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index 7b5575a5f..b9aef3a92 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -13,6 +13,8 @@ defmodule Pleroma.Workers.BackgroundWorker do queue: "background", max_attempts: 1 + use Pleroma.Workers.WorkerHelper, queue: "background" + @impl Oban.Worker def perform(%{"op" => "fetch_initial_posts", "user_id" => user_id}, _job) do user = User.get_cached_by_id(user_id) diff --git a/lib/pleroma/workers/digest_emails_worker.ex b/lib/pleroma/workers/digest_emails_worker.ex new file mode 100644 index 000000000..ca073ce67 --- /dev/null +++ b/lib/pleroma/workers/digest_emails_worker.ex @@ -0,0 +1,21 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.DigestEmailsWorker do + alias Pleroma.User + + # Note: `max_attempts` is intended to be overridden in `new/2` call + use Oban.Worker, + queue: "digest_emails", + max_attempts: 1 + + use Pleroma.Workers.WorkerHelper, queue: "digest_emails" + + @impl Oban.Worker + def perform(%{"op" => "digest_email", "user_id" => user_id}, _job) do + user_id + |> User.get_cached_by_id() + |> Pleroma.DigestEmailWorker.perform() + end +end diff --git a/lib/pleroma/workers/mailer_worker.ex b/lib/pleroma/workers/mailer_worker.ex index 4f73d61bc..a4bd54a6c 100644 --- a/lib/pleroma/workers/mailer_worker.ex +++ b/lib/pleroma/workers/mailer_worker.ex @@ -3,13 +3,13 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.MailerWorker do - alias Pleroma.User - # Note: `max_attempts` is intended to be overridden in `new/2` call use Oban.Worker, queue: "mailer", max_attempts: 1 + use Pleroma.Workers.WorkerHelper, queue: "mailer" + @impl Oban.Worker def perform(%{"op" => "email", "encoded_email" => encoded_email, "config" => config}, _job) do encoded_email @@ -17,10 +17,4 @@ defmodule Pleroma.Workers.MailerWorker do |> :erlang.binary_to_term() |> Pleroma.Emails.Mailer.deliver(config) end - - def perform(%{"op" => "digest_email", "user_id" => user_id}, _job) do - user_id - |> User.get_cached_by_id() - |> Pleroma.DigestEmailWorker.perform() - end end diff --git a/lib/pleroma/workers/publisher_worker.ex b/lib/pleroma/workers/publisher_worker.ex index 5671d2a29..a3ac22635 100644 --- a/lib/pleroma/workers/publisher_worker.ex +++ b/lib/pleroma/workers/publisher_worker.ex @@ -11,6 +11,8 @@ defmodule Pleroma.Workers.PublisherWorker do queue: "federator_outgoing", max_attempts: 1 + use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing" + def backoff(attempt) when is_integer(attempt) do Pleroma.Workers.WorkerHelper.sidekiq_backoff(attempt, 5) end diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index cdce630f2..3cc415ce4 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -10,6 +10,8 @@ defmodule Pleroma.Workers.ReceiverWorker do queue: "federator_incoming", max_attempts: 1 + use Pleroma.Workers.WorkerHelper, queue: "federator_incoming" + @impl Oban.Worker def perform(%{"op" => "incoming_doc", "body" => doc}, _job) do Federator.perform(:incoming_doc, doc) diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex index 4094411ae..936bb64d3 100644 --- a/lib/pleroma/workers/scheduled_activity_worker.ex +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -8,6 +8,8 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do queue: "scheduled_activities", max_attempts: 1 + use Pleroma.Workers.WorkerHelper, queue: "scheduled_activities" + @impl Oban.Worker def perform(%{"op" => "execute", "activity_id" => activity_id}, _job) do Pleroma.ScheduledActivityWorker.perform(:execute, activity_id) diff --git a/lib/pleroma/workers/subscriber_worker.ex b/lib/pleroma/workers/subscriber_worker.ex index 22d1dc956..4fb994554 100644 --- a/lib/pleroma/workers/subscriber_worker.ex +++ b/lib/pleroma/workers/subscriber_worker.ex @@ -12,6 +12,8 @@ defmodule Pleroma.Workers.SubscriberWorker do queue: "federator_outgoing", max_attempts: 1 + use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing" + @impl Oban.Worker def perform(%{"op" => "refresh_subscriptions"}, _job) do Federator.perform(:refresh_subscriptions) diff --git a/lib/pleroma/workers/transmogrifier_worker.ex b/lib/pleroma/workers/transmogrifier_worker.ex index 6f5c1a2f2..6fecc2bf9 100644 --- a/lib/pleroma/workers/transmogrifier_worker.ex +++ b/lib/pleroma/workers/transmogrifier_worker.ex @@ -10,6 +10,8 @@ defmodule Pleroma.Workers.TransmogrifierWorker do queue: "transmogrifier", max_attempts: 1 + use Pleroma.Workers.WorkerHelper, queue: "transmogrifier" + @impl Oban.Worker def perform(%{"op" => "user_upgrade", "user_id" => user_id}, _job) do user = User.get_cached_by_id(user_id) diff --git a/lib/pleroma/workers/web_pusher_worker.ex b/lib/pleroma/workers/web_pusher_worker.ex index 2b1d3b99a..4c2591a5c 100644 --- a/lib/pleroma/workers/web_pusher_worker.ex +++ b/lib/pleroma/workers/web_pusher_worker.ex @@ -11,6 +11,8 @@ defmodule Pleroma.Workers.WebPusherWorker do queue: "web_push", max_attempts: 1 + use Pleroma.Workers.WorkerHelper, queue: "web_push" + @impl Oban.Worker def perform(%{"op" => "web_push", "notification_id" => notification_id}, _job) do notification = Repo.get(Notification, notification_id) diff --git a/lib/pleroma/workers/worker_helper.ex b/lib/pleroma/workers/worker_helper.ex index f9ed2e64d..b12f198d4 100644 --- a/lib/pleroma/workers/worker_helper.ex +++ b/lib/pleroma/workers/worker_helper.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Workers.WorkerHelper do alias Pleroma.Config + alias Pleroma.Workers.WorkerHelper def worker_args(queue) do case Config.get([:workers, :retries, queue]) do @@ -20,4 +21,21 @@ defmodule Pleroma.Workers.WorkerHelper do trunc(backoff) end + + defmacro __using__(opts) do + caller_module = __CALLER__.module + queue = Keyword.fetch!(opts, :queue) + + quote do + def enqueue(op, params, worker_args \\ []) do + params = Map.merge(%{"op" => op}, params) + queue_atom = String.to_atom(unquote(queue)) + worker_args = worker_args ++ WorkerHelper.worker_args(queue_atom) + + unquote(caller_module) + |> apply(:new, [params, worker_args]) + |> Pleroma.Repo.insert() + end + end + end end From b49085c156a6a4449c95c2c315f6250317122735 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 2 Sep 2019 14:57:40 +0300 Subject: [PATCH 13/45] [#1149] Refactoring: GenServer workers renamed to daemons, `use Oban.Worker` moved to helper. --- config/config.exs | 2 +- lib/pleroma/application.ex | 4 ++-- .../activity_expiration_daemon.ex} | 2 +- .../digest_email_daemon.ex} | 2 +- .../scheduled_activity_daemon.ex} | 2 +- lib/pleroma/workers/activity_expiration_worker.ex | 7 +------ lib/pleroma/workers/background_worker.ex | 5 ----- lib/pleroma/workers/digest_emails_worker.ex | 7 +------ lib/pleroma/workers/mailer_worker.ex | 5 ----- lib/pleroma/workers/publisher_worker.ex | 5 ----- lib/pleroma/workers/receiver_worker.ex | 5 ----- lib/pleroma/workers/scheduled_activity_worker.ex | 7 +------ lib/pleroma/workers/subscriber_worker.ex | 5 ----- lib/pleroma/workers/transmogrifier_worker.ex | 5 ----- lib/pleroma/workers/web_pusher_worker.ex | 5 ----- lib/pleroma/workers/worker_helper.ex | 5 +++++ .../activity_expiration_daemon_test.exs} | 2 +- .../digest_email_daemon_test.exs} | 6 +++--- .../scheduled_activity_daemon_test.exs} | 4 ++-- 19 files changed, 20 insertions(+), 65 deletions(-) rename lib/pleroma/{activity_expiration_worker.ex => daemons/activity_expiration_daemon.ex} (96%) rename lib/pleroma/{digest_email_worker.ex => daemons/digest_email_daemon.ex} (96%) rename lib/pleroma/{scheduled_activity_worker.ex => daemons/scheduled_activity_daemon.ex} (96%) rename test/{activity_expiration_worker_test.exs => daemons/activity_expiration_daemon_test.exs} (86%) rename test/{web/digest_email_worker_test.exs => daemons/digest_email_daemon_test.exs} (88%) rename test/{scheduled_activity_worker_test.exs => daemons/scheduled_activity_daemon_test.exs} (82%) diff --git a/config/config.exs b/config/config.exs index 6fb4a0969..b742a650d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -54,7 +54,7 @@ config :pleroma, Pleroma.Repo, scheduled_jobs = with digest_config <- Application.get_env(:pleroma, :email_notifications)[:digest], true <- digest_config[:active] do - [{digest_config[:schedule], {Pleroma.DigestEmailWorker, :perform, []}}] + [{digest_config[:schedule], {Pleroma.Daemons.DigestEmailDaemon, :perform, []}}] else _ -> [] end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index f8f866dbd..0c27027a0 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -36,8 +36,8 @@ defmodule Pleroma.Application do Pleroma.Emoji, Pleroma.Captcha, Pleroma.FlakeId, - Pleroma.ScheduledActivityWorker, - Pleroma.ActivityExpirationWorker + Pleroma.Daemons.ScheduledActivityDaemon, + Pleroma.Daemons.ActivityExpirationDaemon ] ++ cachex_children() ++ hackney_pool_children() ++ diff --git a/lib/pleroma/activity_expiration_worker.ex b/lib/pleroma/daemons/activity_expiration_daemon.ex similarity index 96% rename from lib/pleroma/activity_expiration_worker.ex rename to lib/pleroma/daemons/activity_expiration_daemon.ex index c0820c202..cab7628c4 100644 --- a/lib/pleroma/activity_expiration_worker.ex +++ b/lib/pleroma/daemons/activity_expiration_daemon.ex @@ -2,7 +2,7 @@ # Copyright © 2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.ActivityExpirationWorker do +defmodule Pleroma.Daemons.ActivityExpirationDaemon do alias Pleroma.Activity alias Pleroma.ActivityExpiration alias Pleroma.Config diff --git a/lib/pleroma/digest_email_worker.ex b/lib/pleroma/daemons/digest_email_daemon.ex similarity index 96% rename from lib/pleroma/digest_email_worker.ex rename to lib/pleroma/daemons/digest_email_daemon.ex index 5be7cf26b..462ad2c55 100644 --- a/lib/pleroma/digest_email_worker.ex +++ b/lib/pleroma/daemons/digest_email_daemon.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.DigestEmailWorker do +defmodule Pleroma.Daemons.DigestEmailDaemon do alias Pleroma.Repo alias Pleroma.Workers.DigestEmailsWorker diff --git a/lib/pleroma/scheduled_activity_worker.ex b/lib/pleroma/daemons/scheduled_activity_daemon.ex similarity index 96% rename from lib/pleroma/scheduled_activity_worker.ex rename to lib/pleroma/daemons/scheduled_activity_daemon.ex index c41a542de..aee5f723a 100644 --- a/lib/pleroma/scheduled_activity_worker.ex +++ b/lib/pleroma/daemons/scheduled_activity_daemon.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.ScheduledActivityWorker do +defmodule Pleroma.Daemons.ScheduledActivityDaemon do @moduledoc """ Sends scheduled activities to the job queue. """ diff --git a/lib/pleroma/workers/activity_expiration_worker.ex b/lib/pleroma/workers/activity_expiration_worker.ex index 60dd3feba..4e3e4195f 100644 --- a/lib/pleroma/workers/activity_expiration_worker.ex +++ b/lib/pleroma/workers/activity_expiration_worker.ex @@ -3,11 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.ActivityExpirationWorker do - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: "activity_expiration", - max_attempts: 1 - use Pleroma.Workers.WorkerHelper, queue: "activity_expiration" @impl Oban.Worker @@ -18,6 +13,6 @@ defmodule Pleroma.Workers.ActivityExpirationWorker do }, _job ) do - Pleroma.ActivityExpirationWorker.perform(:execute, activity_expiration_id) + Pleroma.Daemons.ActivityExpirationDaemon.perform(:execute, activity_expiration_id) end end diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index b9aef3a92..082f20ab7 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -8,11 +8,6 @@ defmodule Pleroma.Workers.BackgroundWorker do alias Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy alias Pleroma.Web.OAuth.Token.CleanWorker - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: "background", - max_attempts: 1 - use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker diff --git a/lib/pleroma/workers/digest_emails_worker.ex b/lib/pleroma/workers/digest_emails_worker.ex index ca073ce67..3e5a836d0 100644 --- a/lib/pleroma/workers/digest_emails_worker.ex +++ b/lib/pleroma/workers/digest_emails_worker.ex @@ -5,17 +5,12 @@ defmodule Pleroma.Workers.DigestEmailsWorker do alias Pleroma.User - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: "digest_emails", - max_attempts: 1 - use Pleroma.Workers.WorkerHelper, queue: "digest_emails" @impl Oban.Worker def perform(%{"op" => "digest_email", "user_id" => user_id}, _job) do user_id |> User.get_cached_by_id() - |> Pleroma.DigestEmailWorker.perform() + |> Pleroma.Daemons.DigestEmailDaemon.perform() end end diff --git a/lib/pleroma/workers/mailer_worker.ex b/lib/pleroma/workers/mailer_worker.ex index a4bd54a6c..1b7a0eb3e 100644 --- a/lib/pleroma/workers/mailer_worker.ex +++ b/lib/pleroma/workers/mailer_worker.ex @@ -3,11 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.MailerWorker do - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: "mailer", - max_attempts: 1 - use Pleroma.Workers.WorkerHelper, queue: "mailer" @impl Oban.Worker diff --git a/lib/pleroma/workers/publisher_worker.ex b/lib/pleroma/workers/publisher_worker.ex index a3ac22635..455f7fc7e 100644 --- a/lib/pleroma/workers/publisher_worker.ex +++ b/lib/pleroma/workers/publisher_worker.ex @@ -6,11 +6,6 @@ defmodule Pleroma.Workers.PublisherWorker do alias Pleroma.Activity alias Pleroma.Web.Federator - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: "federator_outgoing", - max_attempts: 1 - use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing" def backoff(attempt) when is_integer(attempt) do diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index 3cc415ce4..83d528a66 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -5,11 +5,6 @@ defmodule Pleroma.Workers.ReceiverWorker do alias Pleroma.Web.Federator - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: "federator_incoming", - max_attempts: 1 - use Pleroma.Workers.WorkerHelper, queue: "federator_incoming" @impl Oban.Worker diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex index 936bb64d3..ca7d53af1 100644 --- a/lib/pleroma/workers/scheduled_activity_worker.ex +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -3,15 +3,10 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.ScheduledActivityWorker do - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: "scheduled_activities", - max_attempts: 1 - use Pleroma.Workers.WorkerHelper, queue: "scheduled_activities" @impl Oban.Worker def perform(%{"op" => "execute", "activity_id" => activity_id}, _job) do - Pleroma.ScheduledActivityWorker.perform(:execute, activity_id) + Pleroma.Daemons.ScheduledActivityDaemon.perform(:execute, activity_id) end end diff --git a/lib/pleroma/workers/subscriber_worker.ex b/lib/pleroma/workers/subscriber_worker.ex index 4fb994554..fc490e300 100644 --- a/lib/pleroma/workers/subscriber_worker.ex +++ b/lib/pleroma/workers/subscriber_worker.ex @@ -7,11 +7,6 @@ defmodule Pleroma.Workers.SubscriberWorker do alias Pleroma.Web.Federator alias Pleroma.Web.Websub - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: "federator_outgoing", - max_attempts: 1 - use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing" @impl Oban.Worker diff --git a/lib/pleroma/workers/transmogrifier_worker.ex b/lib/pleroma/workers/transmogrifier_worker.ex index 6fecc2bf9..b581a2f86 100644 --- a/lib/pleroma/workers/transmogrifier_worker.ex +++ b/lib/pleroma/workers/transmogrifier_worker.ex @@ -5,11 +5,6 @@ defmodule Pleroma.Workers.TransmogrifierWorker do alias Pleroma.User - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: "transmogrifier", - max_attempts: 1 - use Pleroma.Workers.WorkerHelper, queue: "transmogrifier" @impl Oban.Worker diff --git a/lib/pleroma/workers/web_pusher_worker.ex b/lib/pleroma/workers/web_pusher_worker.ex index 4c2591a5c..bea2baffb 100644 --- a/lib/pleroma/workers/web_pusher_worker.ex +++ b/lib/pleroma/workers/web_pusher_worker.ex @@ -6,11 +6,6 @@ defmodule Pleroma.Workers.WebPusherWorker do alias Pleroma.Notification alias Pleroma.Repo - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: "web_push", - max_attempts: 1 - use Pleroma.Workers.WorkerHelper, queue: "web_push" @impl Oban.Worker diff --git a/lib/pleroma/workers/worker_helper.ex b/lib/pleroma/workers/worker_helper.ex index b12f198d4..358efa14a 100644 --- a/lib/pleroma/workers/worker_helper.ex +++ b/lib/pleroma/workers/worker_helper.ex @@ -27,6 +27,11 @@ defmodule Pleroma.Workers.WorkerHelper do queue = Keyword.fetch!(opts, :queue) quote do + # Note: `max_attempts` is intended to be overridden in `new/2` call + use Oban.Worker, + queue: unquote(queue), + max_attempts: 1 + def enqueue(op, params, worker_args \\ []) do params = Map.merge(%{"op" => op}, params) queue_atom = String.to_atom(unquote(queue)) diff --git a/test/activity_expiration_worker_test.exs b/test/daemons/activity_expiration_daemon_test.exs similarity index 86% rename from test/activity_expiration_worker_test.exs rename to test/daemons/activity_expiration_daemon_test.exs index 939d912f1..31f4a70a6 100644 --- a/test/activity_expiration_worker_test.exs +++ b/test/daemons/activity_expiration_daemon_test.exs @@ -10,7 +10,7 @@ defmodule Pleroma.ActivityExpirationWorkerTest do test "deletes an activity" do activity = insert(:note_activity) expiration = insert(:expiration_in_the_past, %{activity_id: activity.id}) - Pleroma.ActivityExpirationWorker.perform(:execute, expiration.id) + Pleroma.Daemons.ActivityExpirationDaemon.perform(:execute, expiration.id) refute Repo.get(Activity, activity.id) end diff --git a/test/web/digest_email_worker_test.exs b/test/daemons/digest_email_daemon_test.exs similarity index 88% rename from test/web/digest_email_worker_test.exs rename to test/daemons/digest_email_daemon_test.exs index 5dfd920fa..3168f3b9a 100644 --- a/test/web/digest_email_worker_test.exs +++ b/test/daemons/digest_email_daemon_test.exs @@ -2,11 +2,11 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.DigestEmailWorkerTest do +defmodule Pleroma.DigestEmailDaemonTest do use Pleroma.DataCase import Pleroma.Factory - alias Pleroma.DigestEmailWorker + alias Pleroma.Daemons.DigestEmailDaemon alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.CommonAPI @@ -23,7 +23,7 @@ defmodule Pleroma.DigestEmailWorkerTest do User.switch_email_notifications(user2, "digest", true) CommonAPI.post(user, %{"status" => "hey @#{user2.nickname}!"}) - DigestEmailWorker.perform() + DigestEmailDaemon.perform() ObanHelpers.perform_all() # Performing job(s) enqueued at previous step ObanHelpers.perform_all() diff --git a/test/scheduled_activity_worker_test.exs b/test/daemons/scheduled_activity_daemon_test.exs similarity index 82% rename from test/scheduled_activity_worker_test.exs rename to test/daemons/scheduled_activity_daemon_test.exs index e3ad1244e..32820b2b7 100644 --- a/test/scheduled_activity_worker_test.exs +++ b/test/daemons/scheduled_activity_daemon_test.exs @@ -2,7 +2,7 @@ # Copyright © 2017-2018 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.ScheduledActivityWorkerTest do +defmodule Pleroma.ScheduledActivityDaemonTest do use Pleroma.DataCase alias Pleroma.ScheduledActivity import Pleroma.Factory @@ -10,7 +10,7 @@ defmodule Pleroma.ScheduledActivityWorkerTest do test "creates a status from the scheduled activity" do user = insert(:user) scheduled_activity = insert(:scheduled_activity, user: user, params: %{status: "hi"}) - Pleroma.ScheduledActivityWorker.perform(:execute, scheduled_activity.id) + Pleroma.Daemons.ScheduledActivityDaemon.perform(:execute, scheduled_activity.id) refute Repo.get(ScheduledActivity, scheduled_activity.id) activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) From 8cbad5500cefbba1e0bb67604960fc331b75b498 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 4 Sep 2019 15:25:12 +0300 Subject: [PATCH 14/45] add tests for activity_pub/utils.ex --- lib/pleroma/user.ex | 1 + lib/pleroma/web/activity_pub/activity_pub.ex | 12 +- lib/pleroma/web/activity_pub/utils.ex | 298 +++++++++---------- test/web/activity_pub/utils_test.exs | 232 ++++++++++++++- 4 files changed, 371 insertions(+), 172 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 29fd6d2ea..424ed772f 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -147,6 +147,7 @@ defmodule Pleroma.User do Cachex.fetch!(:user_cache, key, fn _ -> {:commit, follow_state(user, target)} end) end + @spec set_follow_state_cache(String.t(), String.t(), String.t()) :: {:ok | :error, boolean()} def set_follow_state_cache(user_ap_id, target_ap_id, state) do Cachex.put( :user_cache, diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index eeb826814..39b46a595 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -435,6 +435,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end + @spec block(User.t(), User.t(), String.t() | nil, boolean) :: {:ok, Activity.t() | nil} def block(blocker, blocked, activity_id \\ nil, local \\ true) do outgoing_blocks = Config.get([:activitypub, :outgoing_blocks]) unfollow_blocked = Config.get([:activitypub, :unfollow_blocked]) @@ -463,10 +464,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end + @spec flag(map()) :: {:ok, Activity.t()} | any def flag( %{ actor: actor, - context: context, + context: _context, account: account, statuses: statuses, content: content @@ -478,14 +480,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do additional = params[:additional] || %{} - params = %{ - actor: actor, - context: context, - account: account, - statuses: statuses, - content: content - } - additional = if forward do Map.merge(additional, %{"to" => [], "cc" => [account.ap_id]}) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index c9c0c3763..cf82d1a9b 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -33,50 +33,40 @@ defmodule Pleroma.Web.ActivityPub.Utils do Map.put(params, "actor", get_ap_id(params["actor"])) end - def determine_explicit_mentions(%{"tag" => tag} = _object) when is_list(tag) do - tag - |> Enum.filter(fn x -> is_map(x) end) - |> Enum.filter(fn x -> x["type"] == "Mention" end) - |> Enum.map(fn x -> x["href"] end) + @spec determine_explicit_mentions(map()) :: map() + def determine_explicit_mentions(%{"tag" => tag} = _) when is_list(tag) do + Enum.flat_map(tag, fn + %{"type" => "Mention", "href" => href} -> [href] + _ -> [] + end) end def determine_explicit_mentions(%{"tag" => tag} = object) when is_map(tag) do - Map.put(object, "tag", [tag]) + object + |> Map.put("tag", [tag]) |> determine_explicit_mentions() end def determine_explicit_mentions(_), do: [] + @spec recipient_in_collection(any(), any()) :: boolean() defp recipient_in_collection(ap_id, coll) when is_binary(coll), do: ap_id == coll defp recipient_in_collection(ap_id, coll) when is_list(coll), do: ap_id in coll defp recipient_in_collection(_, _), do: false + @spec recipient_in_message(User.t(), User.t(), map()) :: boolean() def recipient_in_message(%User{ap_id: ap_id} = recipient, %User{} = actor, params) do + addresses = [params["to"], params["cc"], params["bto"], params["bcc"]] + cond do - recipient_in_collection(ap_id, params["to"]) -> - true - - recipient_in_collection(ap_id, params["cc"]) -> - true - - recipient_in_collection(ap_id, params["bto"]) -> - true - - recipient_in_collection(ap_id, params["bcc"]) -> - true - + Enum.any?(addresses, &recipient_in_collection(ap_id, &1)) -> true # if the message is unaddressed at all, then assume it is directly addressed # to the recipient - !params["to"] && !params["cc"] && !params["bto"] && !params["bcc"] -> - true - + Enum.all?(addresses, &is_nil(&1)) -> true # if the message is sent from somebody the user is following, then assume it # is addressed to the recipient - User.following?(recipient, actor) -> - true - - true -> - false + User.following?(recipient, actor) -> true + true -> false end end @@ -188,53 +178,58 @@ defmodule Pleroma.Web.ActivityPub.Utils do Adds an id and a published data if they aren't there, also adds it to an included object """ - def lazy_put_activity_defaults(map, fake \\ false) do - map = - unless fake do - %{data: %{"id" => context}, id: context_id} = create_context(map["context"]) + @spec lazy_put_activity_defaults(map(), boolean) :: map() + def lazy_put_activity_defaults(map, fake \\ false) - map - |> Map.put_new_lazy("id", &generate_activity_id/0) - |> Map.put_new_lazy("published", &make_date/0) - |> Map.put_new("context", context) - |> Map.put_new("context_id", context_id) - else - map - |> Map.put_new("id", "pleroma:fakeid") - |> Map.put_new_lazy("published", &make_date/0) - |> Map.put_new("context", "pleroma:fakecontext") - |> Map.put_new("context_id", -1) - end + def lazy_put_activity_defaults(map, true) do + map + |> Map.put_new("id", "pleroma:fakeid") + |> Map.put_new_lazy("published", &make_date/0) + |> Map.put_new("context", "pleroma:fakecontext") + |> Map.put_new("context_id", -1) + |> lazy_put_object_defaults(true) + end - if is_map(map["object"]) do - object = lazy_put_object_defaults(map["object"], map, fake) - %{map | "object" => object} - else + def lazy_put_activity_defaults(map, _fake) do + %{data: %{"id" => context}, id: context_id} = create_context(map["context"]) + + map + |> Map.put_new_lazy("id", &generate_activity_id/0) + |> Map.put_new_lazy("published", &make_date/0) + |> Map.put_new("context", context) + |> Map.put_new("context_id", context_id) + |> lazy_put_object_defaults(false) + end + + # Adds an id and published date if they aren't there. + # + @spec lazy_put_object_defaults(map(), boolean()) :: map() + defp lazy_put_object_defaults(%{"object" => map} = activity, true) + when is_map(map) do + object = map - end + |> Map.put_new("id", "pleroma:fake_object_id") + |> Map.put_new_lazy("published", &make_date/0) + |> Map.put_new("context", activity["context"]) + |> Map.put_new("context_id", activity["context_id"]) + |> Map.put_new("fake", true) + + %{activity | "object" => object} end - @doc """ - Adds an id and published date if they aren't there. - """ - def lazy_put_object_defaults(map, activity \\ %{}, fake) + defp lazy_put_object_defaults(%{"object" => map} = activity, _) + when is_map(map) do + object = + map + |> Map.put_new_lazy("id", &generate_object_id/0) + |> Map.put_new_lazy("published", &make_date/0) + |> Map.put_new("context", activity["context"]) + |> Map.put_new("context_id", activity["context_id"]) - def lazy_put_object_defaults(map, activity, true = _fake) do - map - |> Map.put_new_lazy("published", &make_date/0) - |> Map.put_new("id", "pleroma:fake_object_id") - |> Map.put_new("context", activity["context"]) - |> Map.put_new("fake", true) - |> Map.put_new("context_id", activity["context_id"]) + %{activity | "object" => object} end - def lazy_put_object_defaults(map, activity, _fake) do - map - |> Map.put_new_lazy("id", &generate_object_id/0) - |> Map.put_new_lazy("published", &make_date/0) - |> Map.put_new("context", activity["context"]) - |> Map.put_new("context_id", activity["context_id"]) - end + defp lazy_put_object_defaults(activity, _), do: activity @doc """ Inserts a full object if it is contained in an activity. @@ -356,23 +351,30 @@ defmodule Pleroma.Web.ActivityPub.Utils do @doc """ Updates a follow activity's state (for locked accounts). """ + @spec update_follow_state_for_all(Activity.t(), String.t()) :: {:ok, Activity} | {:error, any()} def update_follow_state_for_all( %Activity{data: %{"actor" => actor, "object" => object}} = activity, state ) do - try do - Ecto.Adapters.SQL.query!( - Repo, - "UPDATE activities SET data = jsonb_set(data, '{state}', $1) WHERE data->>'type' = 'Follow' AND data->>'actor' = $2 AND data->>'object' = $3 AND data->>'state' = 'pending'", - [state, actor, object] + query = + from(activity in Activity, + where: fragment("data->>'type' = 'Follow'"), + where: fragment("data->>'state' = 'pending'"), + where: fragment("data->>'actor' = ?", ^actor), + where: fragment("data->>'object' = ?", ^object), + update: [ + set: [ + data: fragment("jsonb_set(data, '{state}', ?)", ^state) + ] + ] ) - User.set_follow_state_cache(actor, object, state) - activity = Activity.get_by_id(activity.id) + with {_, _} <- Repo.update_all(query, []), + {_, _} <- User.set_follow_state_cache(actor, object, state), + %Activity{} = activity <- Activity.get_by_id(activity.id) do {:ok, activity} - rescue - e -> - {:error, e} + else + e -> {:error, e} end end @@ -380,9 +382,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do %Activity{data: %{"actor" => actor, "object" => object}} = activity, state ) do - with new_data <- - activity.data - |> Map.put("state", state), + with new_data <- Map.put(activity.data, "state", state), changeset <- Changeset.change(activity, data: new_data), {:ok, activity} <- Repo.update(changeset), _ <- User.set_follow_state_cache(actor, object, state) do @@ -411,27 +411,17 @@ defmodule Pleroma.Web.ActivityPub.Utils do def fetch_latest_follow(%User{ap_id: follower_id}, %User{ap_id: followed_id}) do query = - from( - activity in Activity, - where: - fragment( - "? ->> 'type' = 'Follow'", - activity.data - ), - where: activity.actor == ^follower_id, - # this is to use the index - where: - fragment( - "coalesce((?)->'object'->>'id', (?)->>'object') = ?", - activity.data, - activity.data, - ^followed_id - ), - order_by: [fragment("? desc nulls last", activity.id)], - limit: 1 - ) + follower_id + |> Activity.Queries.by_actor() + |> Activity.Queries.by_type("Follow") + |> Activity.Queries.by_object_id(followed_id) + |> Activity.Queries.limit(1) - Repo.one(query) + from( + activity in query, + order_by: [fragment("? desc nulls last", activity.id)] + ) + |> Repo.one() end #### Announce-related helpers @@ -439,23 +429,14 @@ defmodule Pleroma.Web.ActivityPub.Utils do @doc """ Retruns an existing announce activity if the notice has already been announced """ + @spec get_existing_announce(String.t(), map()) :: Activity.t() | nil def get_existing_announce(actor, %{data: %{"id" => id}}) do - query = - from( - activity in Activity, - where: activity.actor == ^actor, - # this is to use the index - where: - fragment( - "coalesce((?)->'object'->>'id', (?)->>'object') = ?", - activity.data, - activity.data, - ^id - ), - where: fragment("(?)->>'type' = 'Announce'", activity.data) - ) - - Repo.one(query) + actor + |> Activity.Queries.by_actor() + |> Activity.Queries.by_type("Announce") + |> Activity.Queries.by_object_id(id) + |> Activity.Queries.limit(1) + |> Repo.one() end @doc """ @@ -531,31 +512,35 @@ defmodule Pleroma.Web.ActivityPub.Utils do |> maybe_put("id", activity_id) end + @spec add_announce_to_object(Activity.t(), Object.t()) :: + {:ok, Object.t()} | {:error, Ecto.Changeset.t()} def add_announce_to_object( - %Activity{ - data: %{"actor" => actor, "cc" => [Pleroma.Constants.as_public()]} - }, + %Activity{data: %{"actor" => actor, "cc" => [Pleroma.Constants.as_public()]}}, object ) do - announcements = - if is_list(object.data["announcements"]), do: object.data["announcements"], else: [] + announcements = fetch_announcements(object) - with announcements <- [actor | announcements] |> Enum.uniq() do + with announcements <- Enum.uniq([actor | announcements]) do update_element_in_object("announcement", announcements, object) end end def add_announce_to_object(_, object), do: {:ok, object} + @spec remove_announce_from_object(Activity.t(), Object.t()) :: + {:ok, Object.t()} | {:error, Ecto.Changeset.t()} def remove_announce_from_object(%Activity{data: %{"actor" => actor}}, object) do - announcements = - if is_list(object.data["announcements"]), do: object.data["announcements"], else: [] - - with announcements <- announcements |> List.delete(actor) do + with announcements <- List.delete(fetch_announcements(object), actor) do update_element_in_object("announcement", announcements, object) end end + defp fetch_announcements(%{data: %{"announcements" => announcements}} = _) + when is_list(announcements), + do: announcements + + defp fetch_announcements(_), do: [] + #### Unfollow-related helpers def make_unfollow_data(follower, followed, follow_activity, activity_id) do @@ -569,29 +554,20 @@ defmodule Pleroma.Web.ActivityPub.Utils do end #### Block-related helpers + @spec fetch_latest_block(User.t(), User.t()) :: Activity.t() | nil def fetch_latest_block(%User{ap_id: blocker_id}, %User{ap_id: blocked_id}) do query = - from( - activity in Activity, - where: - fragment( - "? ->> 'type' = 'Block'", - activity.data - ), - where: activity.actor == ^blocker_id, - # this is to use the index - where: - fragment( - "coalesce((?)->'object'->>'id', (?)->>'object') = ?", - activity.data, - activity.data, - ^blocked_id - ), - order_by: [fragment("? desc nulls last", activity.id)], - limit: 1 - ) + blocker_id + |> Activity.Queries.by_actor() + |> Activity.Queries.by_type("Block") + |> Activity.Queries.by_object_id(blocked_id) + |> Activity.Queries.limit(1) - Repo.one(query) + from( + activity in query, + order_by: [fragment("? desc nulls last", activity.id)] + ) + |> Repo.one() end def make_block_data(blocker, blocked, activity_id) do @@ -631,28 +607,32 @@ defmodule Pleroma.Web.ActivityPub.Utils do end #### Flag-related helpers - - def make_flag_data(params, additional) do - status_ap_ids = - Enum.map(params.statuses || [], fn - %Activity{} = act -> act.data["id"] - act when is_map(act) -> act["id"] - act when is_binary(act) -> act - end) - - object = [params.account.ap_id] ++ status_ap_ids - + @spec make_flag_data(map(), map()) :: map() + def make_flag_data(%{actor: actor, context: context, content: content} = params, additional) do %{ "type" => "Flag", - "actor" => params.actor.ap_id, - "content" => params.content, - "object" => object, - "context" => params.context, + "actor" => actor.ap_id, + "content" => content, + "object" => build_flag_object(params), + "context" => context, "state" => "open" } |> Map.merge(additional) end + def make_flag_data(_, _), do: %{} + + defp build_flag_object(%{account: account, statuses: statuses} = _) do + [account.ap_id] ++ + Enum.map(statuses || [], fn + %Activity{} = act -> act.data["id"] + act when is_map(act) -> act["id"] + act when is_binary(act) -> act + end) + end + + defp build_flag_object(_), do: [] + @doc """ Fetches the OrderedCollection/OrderedCollectionPage from `from`, limiting the amount of pages fetched after the first one to `pages_left` pages. diff --git a/test/web/activity_pub/utils_test.exs b/test/web/activity_pub/utils_test.exs index eb429b2c4..b1c1d6f71 100644 --- a/test/web/activity_pub/utils_test.exs +++ b/test/web/activity_pub/utils_test.exs @@ -87,6 +87,18 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do assert Utils.determine_explicit_mentions(object) == [] end + + test "works with an object has tags as map" do + object = %{ + "tag" => %{ + "type" => "Mention", + "href" => "https://example.com/~alyssa", + "name" => "Alyssa P. Hacker" + } + } + + assert Utils.determine_explicit_mentions(object) == ["https://example.com/~alyssa"] + end end describe "make_unlike_data/3" do @@ -300,8 +312,8 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do {:ok, follow_activity_two} = Utils.update_follow_state_for_all(follow_activity_two, "accept") - assert Repo.get(Activity, follow_activity.id).data["state"] == "accept" - assert Repo.get(Activity, follow_activity_two.id).data["state"] == "accept" + assert refresh_record(follow_activity).data["state"] == "accept" + assert refresh_record(follow_activity_two).data["state"] == "accept" end end @@ -323,8 +335,8 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do {:ok, follow_activity_two} = Utils.update_follow_state(follow_activity_two, "reject") - assert Repo.get(Activity, follow_activity.id).data["state"] == "pending" - assert Repo.get(Activity, follow_activity_two.id).data["state"] == "reject" + assert refresh_record(follow_activity).data["state"] == "pending" + assert refresh_record(follow_activity_two).data["state"] == "reject" end end @@ -401,4 +413,216 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do assert ^like_activity = Utils.get_existing_like(user.ap_id, object) end end + + describe "get_get_existing_announce/2" do + test "returns nil if announce not found" do + actor = insert(:user) + refute Utils.get_existing_announce(actor.ap_id, %{data: %{"id" => "test"}}) + end + + test "fetches existing announce" do + note_activity = insert(:note_activity) + assert object = Object.normalize(note_activity) + actor = insert(:user) + + {:ok, announce, _object} = ActivityPub.announce(actor, object) + assert Utils.get_existing_announce(actor.ap_id, object) == announce + end + end + + describe "fetch_latest_block/2" do + test "fetches last block activities" do + user1 = insert(:user) + user2 = insert(:user) + + assert {:ok, %Activity{} = _} = ActivityPub.block(user1, user2) + assert {:ok, %Activity{} = _} = ActivityPub.block(user1, user2) + assert {:ok, %Activity{} = activity} = ActivityPub.block(user1, user2) + + assert Utils.fetch_latest_block(user1, user2) == activity + end + end + + describe "recipient_in_message/3" do + test "returns true when recipient in `to`" do + recipient = insert(:user) + actor = insert(:user) + assert Utils.recipient_in_message(recipient, actor, %{"to" => recipient.ap_id}) + + assert Utils.recipient_in_message( + recipient, + actor, + %{"to" => [recipient.ap_id], "cc" => ""} + ) + end + + test "returns true when recipient in `cc`" do + recipient = insert(:user) + actor = insert(:user) + assert Utils.recipient_in_message(recipient, actor, %{"cc" => recipient.ap_id}) + + assert Utils.recipient_in_message( + recipient, + actor, + %{"cc" => [recipient.ap_id], "to" => ""} + ) + end + + test "returns true when recipient in `bto`" do + recipient = insert(:user) + actor = insert(:user) + assert Utils.recipient_in_message(recipient, actor, %{"bto" => recipient.ap_id}) + + assert Utils.recipient_in_message( + recipient, + actor, + %{"bcc" => "", "bto" => [recipient.ap_id]} + ) + end + + test "returns true when recipient in `bcc`" do + recipient = insert(:user) + actor = insert(:user) + assert Utils.recipient_in_message(recipient, actor, %{"bcc" => recipient.ap_id}) + + assert Utils.recipient_in_message( + recipient, + actor, + %{"bto" => "", "bcc" => [recipient.ap_id]} + ) + end + + test "returns true when message without addresses fields" do + recipient = insert(:user) + actor = insert(:user) + assert Utils.recipient_in_message(recipient, actor, %{"bccc" => recipient.ap_id}) + + assert Utils.recipient_in_message( + recipient, + actor, + %{"btod" => "", "bccc" => [recipient.ap_id]} + ) + end + + test "returns false" do + recipient = insert(:user) + actor = insert(:user) + refute Utils.recipient_in_message(recipient, actor, %{"to" => "ap_id"}) + end + end + + describe "lazy_put_activity_defaults/2" do + test "returns map with id and published data" do + note_activity = insert(:note_activity) + object = Object.normalize(note_activity) + res = Utils.lazy_put_activity_defaults(%{"context" => object.data["id"]}) + assert res["context"] == object.data["id"] + assert res["context_id"] == object.id + assert res["id"] + assert res["published"] + end + + test "returns map with fake id and published data" do + assert %{ + "context" => "pleroma:fakecontext", + "context_id" => -1, + "id" => "pleroma:fakeid", + "published" => _ + } = Utils.lazy_put_activity_defaults(%{}, true) + end + + test "returns activity data with object" do + note_activity = insert(:note_activity) + object = Object.normalize(note_activity) + + res = + Utils.lazy_put_activity_defaults(%{ + "context" => object.data["id"], + "object" => %{} + }) + + assert res["context"] == object.data["id"] + assert res["context_id"] == object.id + assert res["id"] + assert res["published"] + assert res["object"]["id"] + assert res["object"]["published"] + assert res["object"]["context"] == object.data["id"] + assert res["object"]["context_id"] == object.id + end + end + + describe "make_flag_data" do + test "returns empty map when params is invalid" do + assert Utils.make_flag_data(%{}, %{}) == %{} + end + + test "returns map with Flag object" do + reporter = insert(:user) + target_account = insert(:user) + {:ok, activity} = CommonAPI.post(target_account, %{"status" => "foobar"}) + context = Utils.generate_context_id() + content = "foobar" + + target_ap_id = target_account.ap_id + activity_ap_id = activity.data["id"] + + res = + Utils.make_flag_data( + %{ + actor: reporter, + context: context, + account: target_account, + statuses: [%{"id" => activity.data["id"]}], + content: content + }, + %{} + ) + + assert %{ + "type" => "Flag", + "content" => ^content, + "context" => ^context, + "object" => [^target_ap_id, ^activity_ap_id], + "state" => "open" + } = res + end + end + + describe "add_announce_to_object/2" do + test "adds actor to announcement" do + user = insert(:user) + object = insert(:note) + + activity = + insert(:note_activity, + data: %{ + "actor" => user.ap_id, + "cc" => [Pleroma.Constants.as_public()] + } + ) + + assert {:ok, updated_object} = Utils.add_announce_to_object(activity, object) + assert updated_object.data["announcements"] == [user.ap_id] + assert updated_object.data["announcement_count"] == 1 + end + end + + describe "remove_announce_from_object/2" do + test "removes actor from announcements" do + user = insert(:user) + user2 = insert(:user) + + object = + insert(:note, + data: %{"announcements" => [user.ap_id, user2.ap_id], "announcement_count" => 2} + ) + + activity = insert(:note_activity, data: %{"actor" => user.ap_id}) + + assert {:ok, updated_object} = Utils.remove_announce_from_object(activity, object) + assert updated_object.data["announcements"] == [user2.ap_id] + assert updated_object.data["announcement_count"] == 1 + end + end end From a890451187f0b1507be96ccf144b18fdb8294dd8 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 4 Sep 2019 17:42:27 +0300 Subject: [PATCH 15/45] fetch_announcements -> take_announcements --- lib/pleroma/web/activity_pub/utils.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index cf82d1a9b..0d87b9220 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -518,7 +518,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do %Activity{data: %{"actor" => actor, "cc" => [Pleroma.Constants.as_public()]}}, object ) do - announcements = fetch_announcements(object) + announcements = take_announcements(object) with announcements <- Enum.uniq([actor | announcements]) do update_element_in_object("announcement", announcements, object) @@ -530,16 +530,16 @@ defmodule Pleroma.Web.ActivityPub.Utils do @spec remove_announce_from_object(Activity.t(), Object.t()) :: {:ok, Object.t()} | {:error, Ecto.Changeset.t()} def remove_announce_from_object(%Activity{data: %{"actor" => actor}}, object) do - with announcements <- List.delete(fetch_announcements(object), actor) do + with announcements <- List.delete(take_announcements(object), actor) do update_element_in_object("announcement", announcements, object) end end - defp fetch_announcements(%{data: %{"announcements" => announcements}} = _) + defp take_announcements(%{data: %{"announcements" => announcements}} = _) when is_list(announcements), do: announcements - defp fetch_announcements(_), do: [] + defp take_announcements(_), do: [] #### Unfollow-related helpers From af746fa4a814dbacd4fe4a3e58b1ee1732363d22 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Wed, 4 Sep 2019 20:08:13 +0300 Subject: [PATCH 16/45] Return total for reports --- CHANGELOG.md | 3 ++- docs/api/admin_api.md | 1 + lib/pleroma/web/admin_api/admin_api_controller.ex | 6 ++---- lib/pleroma/web/admin_api/views/report_view.ex | 3 ++- test/web/admin_api/admin_api_controller_test.exs | 8 ++++++++ 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a414ba5e0..942605f28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Unsubscribe followers when they unfollow a user - AdminAPI: Add "godmode" while fetching user statuses (i.e. admin can see private statuses) - Improve digest email template -– Pagination: (optional) return `total` alongside with `items` when paginating +- Pagination: (optional) return `total` alongside with `items` when paginating +- Admin API: Return `total` when querying for reports ### Fixed - Following from Osada diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index d79c342be..5a090c720 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -313,6 +313,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret ```json { + "total" : 1, "reports": [ { "account": { diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 544b9d7d8..2a1cc59e5 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -442,11 +442,9 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do params |> Map.put("type", "Flag") |> Map.put("skip_preload", true) + |> Map.put("total", true) - reports = - [] - |> ActivityPub.fetch_activities(params) - |> Enum.reverse() + reports = ActivityPub.fetch_activities([], params) conn |> put_view(ReportView) diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index a25f3f1fe..0b8745b2e 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -12,7 +12,8 @@ defmodule Pleroma.Web.AdminAPI.ReportView do def render("index.json", %{reports: reports}) do %{ - reports: render_many(reports, __MODULE__, "show.json", as: :report) + reports: render_many(reports[:items], __MODULE__, "show.json", as: :report), + total: reports[:total] } end diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 4e2c27431..b1ddd898b 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -1309,6 +1309,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do |> json_response(:ok) assert Enum.empty?(response["reports"]) + assert response["total"] == 0 end test "returns reports", %{conn: conn} do @@ -1331,6 +1332,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do assert length(response["reports"]) == 1 assert report["id"] == report_id + + assert response["total"] == 1 end test "returns reports with specified state", %{conn: conn} do @@ -1364,6 +1367,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do assert length(response["reports"]) == 1 assert open_report["id"] == first_report_id + assert response["total"] == 1 + response = conn |> get("/api/pleroma/admin/reports", %{ @@ -1376,6 +1381,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do assert length(response["reports"]) == 1 assert closed_report["id"] == second_report_id + assert response["total"] == 1 + response = conn |> get("/api/pleroma/admin/reports", %{ @@ -1384,6 +1391,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do |> json_response(:ok) assert Enum.empty?(response["reports"]) + assert response["total"] == 0 end test "returns 403 when requested by a non-admin" do From 8306078de1abade082f932cda5b8d9297bdcdc80 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 4 Sep 2019 17:31:14 +0000 Subject: [PATCH 17/45] Apply suggestion to lib/pleroma/web/activity_pub/utils.ex --- lib/pleroma/web/activity_pub/utils.ex | 33 +++++++++++---------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 0d87b9220..2de02f607 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -356,26 +356,19 @@ defmodule Pleroma.Web.ActivityPub.Utils do %Activity{data: %{"actor" => actor, "object" => object}} = activity, state ) do - query = - from(activity in Activity, - where: fragment("data->>'type' = 'Follow'"), - where: fragment("data->>'state' = 'pending'"), - where: fragment("data->>'actor' = ?", ^actor), - where: fragment("data->>'object' = ?", ^object), - update: [ - set: [ - data: fragment("jsonb_set(data, '{state}', ?)", ^state) - ] - ] - ) - - with {_, _} <- Repo.update_all(query, []), - {_, _} <- User.set_follow_state_cache(actor, object, state), - %Activity{} = activity <- Activity.get_by_id(activity.id) do - {:ok, activity} - else - e -> {:error, e} - end + "Follow" + |> Activity.Queries.by_type() + |> Activity.Queries.by_actor(actor) + |> Activity.Queries.by_object_id(object["id"]) + |> where(fragment("data->>'state' = 'pending'")) + |> update(set: [data: fragment("jsonb_set(data, '{state}', ?)", ^state)]) + |> Repo.update_all([]) + + User.set_follow_state_cache(actor, object, state) + + activity = Activity.get_by_id(activity.id) + + {:ok, activity} end def update_follow_state( From e2011a667cdf5e67f257c9c30a02c206fb4df913 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 4 Sep 2019 18:35:01 +0000 Subject: [PATCH 18/45] Apply suggestion to lib/pleroma/web/activity_pub/utils.ex --- lib/pleroma/web/activity_pub/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 2de02f607..011acd48e 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -359,7 +359,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do "Follow" |> Activity.Queries.by_type() |> Activity.Queries.by_actor(actor) - |> Activity.Queries.by_object_id(object["id"]) + |> Activity.Queries.by_object_id(object) |> where(fragment("data->>'state' = 'pending'")) |> update(set: [data: fragment("jsonb_set(data, '{state}', ?)", ^state)]) |> Repo.update_all([]) From ae506ca997619f118d18703a9b0802246eb427d5 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 4 Sep 2019 21:40:53 +0300 Subject: [PATCH 19/45] fix formatting --- lib/pleroma/web/activity_pub/utils.ex | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 011acd48e..72e07b59d 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -356,19 +356,19 @@ defmodule Pleroma.Web.ActivityPub.Utils do %Activity{data: %{"actor" => actor, "object" => object}} = activity, state ) do - "Follow" - |> Activity.Queries.by_type() - |> Activity.Queries.by_actor(actor) - |> Activity.Queries.by_object_id(object) - |> where(fragment("data->>'state' = 'pending'")) - |> update(set: [data: fragment("jsonb_set(data, '{state}', ?)", ^state)]) - |> Repo.update_all([]) - - User.set_follow_state_cache(actor, object, state) - - activity = Activity.get_by_id(activity.id) - - {:ok, activity} + "Follow" + |> Activity.Queries.by_type() + |> Activity.Queries.by_actor(actor) + |> Activity.Queries.by_object_id(object) + |> where(fragment("data->>'state' = 'pending'")) + |> update(set: [data: fragment("jsonb_set(data, '{state}', ?)", ^state)]) + |> Repo.update_all([]) + + User.set_follow_state_cache(actor, object, state) + + activity = Activity.get_by_id(activity.id) + + {:ok, activity} end def update_follow_state( From 736165c082d34ef4d52367ea8315c228a1df3944 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Thu, 5 Sep 2019 16:54:34 +0300 Subject: [PATCH 20/45] Reverse reports list --- lib/pleroma/web/admin_api/views/report_view.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index 0b8745b2e..51b95ad5e 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -12,7 +12,8 @@ defmodule Pleroma.Web.AdminAPI.ReportView do def render("index.json", %{reports: reports}) do %{ - reports: render_many(reports[:items], __MODULE__, "show.json", as: :report), + reports: + render_many(reports[:items], __MODULE__, "show.json", as: :report) |> Enum.reverse(), total: reports[:total] } end From 769fb778d41df77c2514b5e3c663f3f624c0a266 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 12 Sep 2019 21:37:36 +0300 Subject: [PATCH 21/45] Track object/create activity fetches --- lib/pleroma/delivery.ex | 58 +++++++++++++ lib/pleroma/plugs/cache.ex | 16 +++- lib/pleroma/user.ex | 10 +++ .../activity_pub/activity_pub_controller.ex | 29 ++++++- .../20190912065617_create_deliveries.exs | 12 +++ .../activity_pub_controller_test.exs | 83 +++++++++++++++++++ 6 files changed, 206 insertions(+), 2 deletions(-) create mode 100644 lib/pleroma/delivery.ex create mode 100644 priv/repo/migrations/20190912065617_create_deliveries.exs diff --git a/lib/pleroma/delivery.ex b/lib/pleroma/delivery.ex new file mode 100644 index 000000000..f9a9e35cd --- /dev/null +++ b/lib/pleroma/delivery.ex @@ -0,0 +1,58 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Delivery do + use Ecto.Schema + + alias Pleroma.Delivery + alias Pleroma.FlakeId + alias Pleroma.User + alias Pleroma.Repo + alias Pleroma.Object + alias Pleroma.User + + import Ecto.Changeset + import Ecto.Query + + schema "deliveries" do + belongs_to(:user, User, type: FlakeId) + belongs_to(:object, Object) + end + + def changeset(delivery, params \\ %{}) do + delivery + |> cast(params, [:user_id, :object_id]) + |> foreign_key_constraint(:object_id) + |> foreign_key_constraint(:user_id) + |> unique_constraint(:user_id, name: :deliveries_user_id_object_id_index) + end + + def create(object_id, user_id) do + %Delivery{} + |> changeset(%{user_id: user_id, object_id: object_id}) + |> Repo.insert() + end + + def get(object_id, user_id) do + from(d in Delivery, where: d.user_id == ^user_id and d.object_id == ^object_id) + |> Repo.one() + end + + def get_or_create(object_id, user_id) do + case get(object_id, user_id) do + %Delivery{} = delivery -> {:ok, delivery} + nil -> create(object_id, user_id) + end + end + + def delete_all_by_object_id(object_id) do + from(d in Delivery, where: d.object_id == ^object_id) + |> Repo.delete_all() + end + + def get_all_by_object_id(object_id) do + from(d in Delivery, where: d.object_id == ^object_id) + |> Repo.all() + end +end diff --git a/lib/pleroma/plugs/cache.ex b/lib/pleroma/plugs/cache.ex index a81a861d0..42d77fc1f 100644 --- a/lib/pleroma/plugs/cache.ex +++ b/lib/pleroma/plugs/cache.ex @@ -20,6 +20,7 @@ defmodule Pleroma.Plugs.Cache do - `ttl`: An expiration time (time-to-live). This value should be in milliseconds or `nil` to disable expiration. Defaults to `nil`. - `query_params`: Take URL query string into account (`true`), ignore it (`false`) or limit to specific params only (list). Defaults to `true`. + - `tracking_fun`: A function that is called on successfull responses, no matter if the request is cached or not. It should accept a conn as the first argument and the value assigned to `tracking_fun_data` as the second. Additionally, you can overwrite the TTL inside a controller action by assigning `cache_ttl` to the connection struct: @@ -56,6 +57,10 @@ defmodule Pleroma.Plugs.Cache do {:ok, nil} -> cache_resp(conn, opts) + {:ok, {content_type, body, tracking_fun_data}} -> + conn = opts.tracking_fun(conn, tracking_fun_data) + send_cached(conn, {content_type, body}) + {:ok, record} -> send_cached(conn, record) @@ -90,7 +95,16 @@ defmodule Pleroma.Plugs.Cache do content_type = content_type(conn) record = {content_type, body} - Cachex.put(:web_resp_cache, key, record, ttl: ttl) + conn = + unless opts[:tracking_fun] do + Cachex.put(:web_resp_cache, key, {content_type, body}, ttl: ttl) + conn + else + tracking_fun_data = Map.get(conn.assigns, :tracking_fun_data, nil) + Cachex.put(:web_resp_cache, {content_type, body, tracking_fun_data}, record, ttl: ttl) + + opts.tracking_fun.(conn, tracking_fun_data) + end put_resp_header(conn, "x-cache", "MISS from Pleroma") diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3aa245f2a..9614acdab 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -11,6 +11,7 @@ defmodule Pleroma.User do alias Comeonin.Pbkdf2 alias Ecto.Multi alias Pleroma.Activity + alias Pleroma.Delivery alias Pleroma.Keys alias Pleroma.Notification alias Pleroma.Object @@ -61,6 +62,7 @@ defmodule Pleroma.User do field(:last_digest_emailed_at, :naive_datetime) has_many(:notifications, Notification) has_many(:registrations, Registration) + has_many(:deliveries, Delivery) embeds_one(:info, User.Info) timestamps() @@ -1624,4 +1626,12 @@ defmodule Pleroma.User do def is_internal_user?(%User{nickname: nil}), do: true def is_internal_user?(%User{local: true, nickname: "internal." <> _}), do: true def is_internal_user?(_), do: false + + def get_delivered_users_by_object_id(object_id) do + from(u in User, + inner_join: delivery in assoc(u, :deliveries), + where: delivery.object_id == ^object_id + ) + |> Repo.all() + end end diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 705dbc1c2..009260d3f 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do use Pleroma.Web, :controller alias Pleroma.Activity + alias Pleroma.Delivery alias Pleroma.Object alias Pleroma.Object.Fetcher alias Pleroma.User @@ -23,7 +24,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do action_fallback(:errors) - plug(Pleroma.Plugs.Cache, [query_params: false] when action in [:activity, :object]) + plug( + Pleroma.Plugs.Cache, + [ + query_params: false, + tracking_fun: &Pleroma.Web.ActivityPub.ActivityPubController.track_object_fetch/2 + ] + when action in [:activity, :object] + ) + plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay]) plug(:set_requester_reachable when action in [:inbox]) plug(:relay_active? when action in [:relay]) @@ -54,6 +63,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do %Object{} = object <- Object.get_cached_by_ap_id(ap_id), {_, true} <- {:public?, Visibility.is_public?(object)} do conn + |> assign(:tracking_fun_data, object.id) |> set_cache_ttl_for(object) |> put_resp_content_type("application/activity+json") |> put_view(ObjectView) @@ -64,6 +74,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end end + def track_object_fetch(conn, object_id) do + case conn.assigns[:user] do + %User{id: user_id} -> Delivery.create(object_id, user_id) + _ -> nil + end + + conn + end + def object_likes(conn, %{"uuid" => uuid, "page" => page}) do with ap_id <- o_status_url(conn, :object, uuid), %Object{} = object <- Object.get_cached_by_ap_id(ap_id), @@ -99,6 +118,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do %Activity{} = activity <- Activity.normalize(ap_id), {_, true} <- {:public?, Visibility.is_public?(activity)} do conn + |> maybe_set_tracking_data(activity) |> set_cache_ttl_for(activity) |> put_resp_content_type("application/activity+json") |> put_view(ObjectView) @@ -109,6 +129,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end end + defp maybe_set_tracking_data(conn, %Activity{data: %{"type" => "Create"}} = activity) do + object_id = Object.normalize(activity).id + assign(conn, :tracking_fun_data, object_id) + end + + defp maybe_set_tracking_data(conn, _activity), do: assign(conn, :tracking_fun_data, nil) + defp set_cache_ttl_for(conn, %Activity{object: object}) do set_cache_ttl_for(conn, object) end diff --git a/priv/repo/migrations/20190912065617_create_deliveries.exs b/priv/repo/migrations/20190912065617_create_deliveries.exs new file mode 100644 index 000000000..92ca5650a --- /dev/null +++ b/priv/repo/migrations/20190912065617_create_deliveries.exs @@ -0,0 +1,12 @@ +defmodule Pleroma.Repo.Migrations.CreateDeliveries do + use Ecto.Migration + + def change do + create_if_not_exists table(:deliveries) do + add(:object_id, references(:objects, type: :id)) + add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) + end + create_if_not_exists index(:deliveries, :object_id, name: :deliveries_object_id) + create_if_not_exists(unique_index(:deliveries, [:user_id, :object_id])) + end +end diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 9698c7099..0bab555b5 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do use Pleroma.Web.ConnCase import Pleroma.Factory alias Pleroma.Activity + alias Pleroma.Delivery alias Pleroma.Instances alias Pleroma.Object alias Pleroma.User @@ -885,4 +886,86 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert result["totalItems"] == 15 end end + + describe "delivery tracking" do + test "it tracks a signed object fetch", %{conn: conn} do + user = insert(:user, local: false) + activity = insert(:note_activity) + object = Object.normalize(activity) + + object_path = String.trim_leading(object.data["id"], Pleroma.Web.Endpoint.url()) + + conn + |> put_req_header("accept", "application/activity+json") + |> assign(:user, user) + |> get(object_path) + |> json_response(200) + + assert Delivery.get(object.id, user.id) + end + + test "it tracks a signed activity fetch", %{conn: conn} do + user = insert(:user, local: false) + activity = insert(:note_activity) + object = Object.normalize(activity) + + activity_path = String.trim_leading(activity.data["id"], Pleroma.Web.Endpoint.url()) + + conn + |> put_req_header("accept", "application/activity+json") + |> assign(:user, user) + |> get(activity_path) + |> json_response(200) + + assert Delivery.get(object.id, user.id) + end + + test "it tracks a signed object fetch when the json is cached", %{conn: conn} do + user = insert(:user, local: false) + other_user = insert(:user, local: false) + activity = insert(:note_activity) + object = Object.normalize(activity) + + object_path = String.trim_leading(object.data["id"], Pleroma.Web.Endpoint.url()) + + conn + |> put_req_header("accept", "application/activity+json") + |> assign(:user, user) + |> get(object_path) + |> json_response(200) + + build_conn() + |> put_req_header("accept", "application/activity+json") + |> assign(:user, other_user) + |> get(object_path) + |> json_response(200) + + assert Delivery.get(object.id, user.id) + assert Delivery.get(object.id, other_user.id) + end + + test "it tracks a signed activity fetch when the json is cached", %{conn: conn} do + user = insert(:user, local: false) + other_user = insert(:user, local: false) + activity = insert(:note_activity) + object = Object.normalize(activity) + + activity_path = String.trim_leading(activity.data["id"], Pleroma.Web.Endpoint.url()) + + conn + |> put_req_header("accept", "application/activity+json") + |> assign(:user, user) + |> get(activity_path) + |> json_response(200) + + build_conn() + |> put_req_header("accept", "application/activity+json") + |> assign(:user, other_user) + |> get(activity_path) + |> json_response(200) + + assert Delivery.get(object.id, user.id) + assert Delivery.get(object.id, other_user.id) + end + end end From dabc4a00f5cf08dac75f701457a24fce8735175f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 12 Sep 2019 22:10:15 +0300 Subject: [PATCH 22/45] Put the cache with the right key when using a tracking function --- lib/pleroma/plugs/cache.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/plugs/cache.ex b/lib/pleroma/plugs/cache.ex index 42d77fc1f..50b534e7b 100644 --- a/lib/pleroma/plugs/cache.ex +++ b/lib/pleroma/plugs/cache.ex @@ -58,7 +58,8 @@ defmodule Pleroma.Plugs.Cache do cache_resp(conn, opts) {:ok, {content_type, body, tracking_fun_data}} -> - conn = opts.tracking_fun(conn, tracking_fun_data) + conn = opts.tracking_fun.(conn, tracking_fun_data) + send_cached(conn, {content_type, body}) {:ok, record} -> @@ -93,7 +94,6 @@ defmodule Pleroma.Plugs.Cache do ttl = Map.get(conn.assigns, :cache_ttl, opts.ttl) key = cache_key(conn, opts) content_type = content_type(conn) - record = {content_type, body} conn = unless opts[:tracking_fun] do @@ -101,7 +101,7 @@ defmodule Pleroma.Plugs.Cache do conn else tracking_fun_data = Map.get(conn.assigns, :tracking_fun_data, nil) - Cachex.put(:web_resp_cache, {content_type, body, tracking_fun_data}, record, ttl: ttl) + Cachex.put(:web_resp_cache, key, {content_type, body, tracking_fun_data}, ttl: ttl) opts.tracking_fun.(conn, tracking_fun_data) end From b0e60580215e26caae6452099fa1fbace525937c Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 12 Sep 2019 22:40:53 +0300 Subject: [PATCH 23/45] Parse http signature for request to objects/activities --- lib/pleroma/plugs/http_signature.ex | 3 ++- lib/pleroma/web/router.ex | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/plugs/http_signature.ex b/lib/pleroma/plugs/http_signature.ex index d87fa52fa..23d22a712 100644 --- a/lib/pleroma/plugs/http_signature.ex +++ b/lib/pleroma/plugs/http_signature.ex @@ -15,7 +15,8 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do end def call(conn, _opts) do - [signature | _] = get_req_header(conn, "signature") + headers = get_req_header(conn, "signature") + signature = Enum.at(headers, 0) if signature do # set (request-target) header to the appropriate value diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 7cd59acb2..badc7e048 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -135,6 +135,7 @@ defmodule Pleroma.Web.Router do pipeline :http_signature do plug(Pleroma.Web.Plugs.HTTPSignaturePlug) + plug(Pleroma.Web.Plugs.MappedSignatureToIdentityPlug) end scope "/api/pleroma", Pleroma.Web.TwitterAPI do @@ -513,6 +514,7 @@ defmodule Pleroma.Web.Router do scope "/", Pleroma.Web do pipe_through(:ostatus) + pipe_through(:http_signature) get("/objects/:uuid", OStatus.OStatusController, :object) get("/activities/:uuid", OStatus.OStatusController, :activity) From ce23529d917c1830b270a29e774e4ed7768bfeff Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 13 Sep 2019 11:36:49 +0300 Subject: [PATCH 24/45] Use delivery info when federating deletes --- lib/pleroma/delivery.ex | 4 ++ lib/pleroma/user.ex | 4 ++ lib/pleroma/web/activity_pub/publisher.ex | 15 ++++- test/web/activity_pub/publisher_test.exs | 68 ++++++++++++++++++++++- 4 files changed, 89 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/delivery.ex b/lib/pleroma/delivery.ex index f9a9e35cd..2e7c019fa 100644 --- a/lib/pleroma/delivery.ex +++ b/lib/pleroma/delivery.ex @@ -46,6 +46,10 @@ defmodule Pleroma.Delivery do end end + # A hack because user delete activities have a fake id for whatever reason + # TODO: Get rid of this + def delete_all_by_object_id("pleroma:fake_object_id"), do: {0, []} + def delete_all_by_object_id(object_id) do from(d in Delivery, where: d.object_id == ^object_id) |> Repo.delete_all() diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 9614acdab..785b22643 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1627,6 +1627,10 @@ defmodule Pleroma.User do def is_internal_user?(%User{local: true, nickname: "internal." <> _}), do: true def is_internal_user?(_), do: false + # A hack because user delete activities have a fake id for whatever reason + # TODO: Get rid of this + def get_delivered_users_by_object_id("pleroma:fake_object_id"), do: [] + def get_delivered_users_by_object_id(object_id) do from(u in User, inner_join: delivery in assoc(u, :deliveries), diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index c97405690..db64fd2f6 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -5,9 +5,11 @@ defmodule Pleroma.Web.ActivityPub.Publisher do alias Pleroma.Activity alias Pleroma.Config + alias Pleroma.Delivery alias Pleroma.HTTP alias Pleroma.Instances alias Pleroma.User + alias Pleroma.Object alias Pleroma.Web.ActivityPub.Relay alias Pleroma.Web.ActivityPub.Transmogrifier @@ -107,7 +109,18 @@ defmodule Pleroma.Web.ActivityPub.Publisher do {:ok, []} end - Pleroma.Web.Salmon.remote_users(actor, activity) ++ followers + fetchers = + with %Activity{data: %{"type" => "Delete"}} <- activity, + %Object{id: object_id} <- Object.normalize(activity), + fetchers <- User.get_delivered_users_by_object_id(object_id), + _ <- Delivery.delete_all_by_object_id(object_id) do + fetchers + else + _ -> + [] + end + + Pleroma.Web.Salmon.remote_users(actor, activity) ++ followers ++ fetchers end defp get_cc_ap_ids(ap_id, recipients) do diff --git a/test/web/activity_pub/publisher_test.exs b/test/web/activity_pub/publisher_test.exs index 36a39c84c..32b7a242c 100644 --- a/test/web/activity_pub/publisher_test.exs +++ b/test/web/activity_pub/publisher_test.exs @@ -3,15 +3,17 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.PublisherTest do - use Pleroma.DataCase + use Pleroma.Web.ConnCase import Pleroma.Factory import Tesla.Mock import Mock alias Pleroma.Activity + alias Pleroma.Object alias Pleroma.Instances alias Pleroma.Web.ActivityPub.Publisher + alias Pleroma.Web.CommonAPI @as_public "https://www.w3.org/ns/activitystreams#Public" @@ -262,5 +264,69 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do }) ) end + + test_with_mock "publishes a delete activity to peers who signed fetch requests to the create acitvity/object.", + Pleroma.Web.Federator.Publisher, + [:passthrough], + [] do + fetcher = + insert(:user, + local: false, + info: %{ + ap_enabled: true, + source_data: %{"inbox" => "https://domain.com/users/nick1/inbox"} + } + ) + + another_fetcher = + insert(:user, + local: false, + info: %{ + ap_enabled: true, + source_data: %{"inbox" => "https://domain2.com/users/nick1/inbox"} + } + ) + + actor = insert(:user) + + note_activity = insert(:note_activity, user: actor) + object = Object.normalize(note_activity) + + activity_path = String.trim_leading(note_activity.data["id"], Pleroma.Web.Endpoint.url()) + object_path = String.trim_leading(object.data["id"], Pleroma.Web.Endpoint.url()) + + build_conn() + |> put_req_header("accept", "application/activity+json") + |> assign(:user, fetcher) + |> get(object_path) + |> json_response(200) + + build_conn() + |> put_req_header("accept", "application/activity+json") + |> assign(:user, another_fetcher) + |> get(activity_path) + |> json_response(200) + + {:ok, delete} = CommonAPI.delete(note_activity.id, actor) + + res = Publisher.publish(actor, delete) + assert res == :ok + + assert called( + Pleroma.Web.Federator.Publisher.enqueue_one(Publisher, %{ + inbox: "https://domain.com/users/nick1/inbox", + actor: actor, + id: delete.data["id"] + }) + ) + + assert called( + Pleroma.Web.Federator.Publisher.enqueue_one(Publisher, %{ + inbox: "https://domain2.com/users/nick1/inbox", + actor: actor, + id: delete.data["id"] + }) + ) + end end end From fb96facc32fb275efffeefa2892a1098ecd68b77 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 13 Sep 2019 12:06:31 +0300 Subject: [PATCH 25/45] Remove unused functions and fix credo issues --- lib/pleroma/delivery.ex | 16 ++-------------- lib/pleroma/web/activity_pub/publisher.ex | 2 +- test/web/activity_pub/publisher_test.exs | 2 +- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/delivery.ex b/lib/pleroma/delivery.ex index 2e7c019fa..ce8fb96f4 100644 --- a/lib/pleroma/delivery.ex +++ b/lib/pleroma/delivery.ex @@ -7,9 +7,9 @@ defmodule Pleroma.Delivery do alias Pleroma.Delivery alias Pleroma.FlakeId - alias Pleroma.User - alias Pleroma.Repo alias Pleroma.Object + alias Pleroma.Repo + alias Pleroma.User alias Pleroma.User import Ecto.Changeset @@ -39,13 +39,6 @@ defmodule Pleroma.Delivery do |> Repo.one() end - def get_or_create(object_id, user_id) do - case get(object_id, user_id) do - %Delivery{} = delivery -> {:ok, delivery} - nil -> create(object_id, user_id) - end - end - # A hack because user delete activities have a fake id for whatever reason # TODO: Get rid of this def delete_all_by_object_id("pleroma:fake_object_id"), do: {0, []} @@ -54,9 +47,4 @@ defmodule Pleroma.Delivery do from(d in Delivery, where: d.object_id == ^object_id) |> Repo.delete_all() end - - def get_all_by_object_id(object_id) do - from(d in Delivery, where: d.object_id == ^object_id) - |> Repo.all() - end end diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index db64fd2f6..c39e89a6a 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -8,8 +8,8 @@ defmodule Pleroma.Web.ActivityPub.Publisher do alias Pleroma.Delivery alias Pleroma.HTTP alias Pleroma.Instances - alias Pleroma.User alias Pleroma.Object + alias Pleroma.User alias Pleroma.Web.ActivityPub.Relay alias Pleroma.Web.ActivityPub.Transmogrifier diff --git a/test/web/activity_pub/publisher_test.exs b/test/web/activity_pub/publisher_test.exs index 32b7a242c..0ef97464e 100644 --- a/test/web/activity_pub/publisher_test.exs +++ b/test/web/activity_pub/publisher_test.exs @@ -10,8 +10,8 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do import Mock alias Pleroma.Activity - alias Pleroma.Object alias Pleroma.Instances + alias Pleroma.Object alias Pleroma.Web.ActivityPub.Publisher alias Pleroma.Web.CommonAPI From 517017048316a52172d60d26b03beddb85af7b39 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 13 Sep 2019 10:09:46 +0000 Subject: [PATCH 26/45] Apply suggestion to lib/pleroma/web/activity_pub/activity_pub_controller.ex --- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 009260d3f..025641722 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -26,10 +26,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do plug( Pleroma.Plugs.Cache, - [ - query_params: false, - tracking_fun: &Pleroma.Web.ActivityPub.ActivityPubController.track_object_fetch/2 - ] + [query_params: false, tracking_fun: &__MODULE__.track_object_fetch/2] when action in [:activity, :object] ) From 3896a51b8aefe6fe54251ffd559c636980faa87e Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 13 Sep 2019 10:09:56 +0000 Subject: [PATCH 27/45] Apply suggestion to lib/pleroma/web/activity_pub/activity_pub_controller.ex --- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 025641722..4bd13defb 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -72,9 +72,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end def track_object_fetch(conn, object_id) do - case conn.assigns[:user] do - %User{id: user_id} -> Delivery.create(object_id, user_id) - _ -> nil + with %{assigns: %{user: %User{id: user_id}}} <- conn do + Delivery.create(object_id, user_id) end conn From 2784962dba295ee35677e93996df53d1711e5768 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 13 Sep 2019 15:23:03 +0000 Subject: [PATCH 28/45] Apply suggestion to lib/pleroma/web/activity_pub/activity_pub_controller.ex --- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 4bd13defb..70d4a5baf 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -130,7 +130,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do assign(conn, :tracking_fun_data, object_id) end - defp maybe_set_tracking_data(conn, _activity), do: assign(conn, :tracking_fun_data, nil) + defp maybe_set_tracking_data(conn, _activity), do: conn defp set_cache_ttl_for(conn, %Activity{object: object}) do set_cache_ttl_for(conn, object) From 05f8a066a107af2f7151aee8d85af97cf6a4835c Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 13 Sep 2019 15:23:26 +0000 Subject: [PATCH 29/45] Apply suggestion to lib/pleroma/delivery.ex --- lib/pleroma/delivery.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/delivery.ex b/lib/pleroma/delivery.ex index ce8fb96f4..38c148c34 100644 --- a/lib/pleroma/delivery.ex +++ b/lib/pleroma/delivery.ex @@ -31,7 +31,7 @@ defmodule Pleroma.Delivery do def create(object_id, user_id) do %Delivery{} |> changeset(%{user_id: user_id, object_id: object_id}) - |> Repo.insert() + |> Repo.insert(on_conflict: :nothing) end def get(object_id, user_id) do From 8900cb68aef535dbf60de87fce47d85b91909077 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 13 Sep 2019 15:25:15 +0000 Subject: [PATCH 30/45] Apply suggestion to lib/pleroma/web/activity_pub/activity_pub_controller.ex --- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 70d4a5baf..01b34fb1d 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -71,6 +71,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end end + def track_object_fetch(conn, nil), do: conn + def track_object_fetch(conn, object_id) do with %{assigns: %{user: %User{id: user_id}}} <- conn do Delivery.create(object_id, user_id) From ac4a748fad34c02647bf72e802cd9d74205681fe Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 13 Sep 2019 19:28:35 +0300 Subject: [PATCH 31/45] Disallow NULLs in deliveries --- lib/pleroma/delivery.ex | 1 + priv/repo/migrations/20190912065617_create_deliveries.exs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/delivery.ex b/lib/pleroma/delivery.ex index 38c148c34..29a1e5a77 100644 --- a/lib/pleroma/delivery.ex +++ b/lib/pleroma/delivery.ex @@ -23,6 +23,7 @@ defmodule Pleroma.Delivery do def changeset(delivery, params \\ %{}) do delivery |> cast(params, [:user_id, :object_id]) + |> validate_required([:user_id, :object_id]) |> foreign_key_constraint(:object_id) |> foreign_key_constraint(:user_id) |> unique_constraint(:user_id, name: :deliveries_user_id_object_id_index) diff --git a/priv/repo/migrations/20190912065617_create_deliveries.exs b/priv/repo/migrations/20190912065617_create_deliveries.exs index 92ca5650a..79071a799 100644 --- a/priv/repo/migrations/20190912065617_create_deliveries.exs +++ b/priv/repo/migrations/20190912065617_create_deliveries.exs @@ -3,8 +3,8 @@ defmodule Pleroma.Repo.Migrations.CreateDeliveries do def change do create_if_not_exists table(:deliveries) do - add(:object_id, references(:objects, type: :id)) - add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) + add(:object_id, references(:objects, type: :id), null: false) + add(:user_id, references(:users, type: :uuid, on_delete: :delete_all), null: false) end create_if_not_exists index(:deliveries, :object_id, name: :deliveries_object_id) create_if_not_exists(unique_index(:deliveries, [:user_id, :object_id])) From e127b9ab6d01da48ebad188d2b9fcf7cb8a41578 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 14 Sep 2019 16:28:59 +0300 Subject: [PATCH 32/45] [#1149] Rewritten readme as config/description.exs. --- config/description.exs | 198 ++++++++++++++++++++++++++--------------- 1 file changed, 126 insertions(+), 72 deletions(-) diff --git a/config/description.exs b/config/description.exs index c5ae63915..be5eb0cc3 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1778,87 +1778,141 @@ config :pleroma, :config_description, [ group: :pleroma_job_queue, key: :queues, type: :group, - description: "Pleroma Job Queue configuration: a list of queues with maximum concurrent jobs", - children: [ - %{ - key: :federator_outgoing, - type: :integer, - description: "Outgoing federation queue", - suggestions: [50] - }, - %{ - key: :federator_incoming, - type: :integer, - description: "Incoming federation queue", - suggestions: [50] - }, - %{ - key: :mailer, - type: :integer, - description: "Email sender queue, see Pleroma.Emails.Mailer", - suggestions: [10] - }, - %{ - key: :web_push, - type: :integer, - description: "Web push notifications queue", - suggestions: [50] - }, - %{ - key: :transmogrifier, - type: :integer, - description: "Transmogrifier queue", - suggestions: [20] - }, - %{ - key: :scheduled_activities, - type: :integer, - description: "Scheduled activities queue, see Pleroma.ScheduledActivities", - suggestions: [10] - }, - %{ - key: :activity_expiration, - type: :integer, - description: "Activity expiration queue", - suggestions: [10] - }, - %{ - key: :background, - type: :integer, - description: "Background queue", - suggestions: [5] - } - ] + description: "[Deprecated] Replaced with `Oban`/`:queues` (keeping the same format)", + children: [] }, %{ group: :pleroma, key: Pleroma.Web.Federator.RetryQueue, type: :group, - description: "", + description: "[Deprecated] See `Oban` and `:workers` sections for configuration notes", children: [ - %{ - key: :enabled, - type: :boolean, - description: "If set to true, failed federation jobs will be retried", - suggestions: [true, false] - }, - %{ - key: :max_jobs, - type: :integer, - description: "The maximum amount of parallel federation jobs running at the same time", - suggestions: [20] - }, - %{ - key: :initial_timeout, - type: :integer, - description: "The initial timeout in seconds", - suggestions: [30] - }, %{ key: :max_retries, type: :integer, - description: "The maximum number of times a federation job is retried", - suggestions: [5] + description: "[Deprecated] Replaced as `Oban`/`:queues`/`:outgoing_federation` value", + suggestions: [] + } + ] + }, + %{ + group: :pleroma, + key: Oban, + type: :group, + description: """ + [Oban](https://github.com/sorentwo/oban) asynchronous job processor configuration. + + Note: if you are running PostgreSQL in [`silent_mode`](https://postgresqlco.nf/en/doc/param/silent_mode?version=9.1), + it's advised to set [`log_destination`](https://postgresqlco.nf/en/doc/param/log_destination?version=9.1) to `syslog`, + otherwise `postmaster.log` file may grow because of "you don't own a lock of type ShareLock" warnings + (see https://github.com/sorentwo/oban/issues/52). + """, + children: [ + %{ + key: :repo, + type: :module, + description: "Application's Ecto repo", + suggestions: [Pleroma.Repo] + }, + %{ + key: :verbose, + type: :boolean, + description: "Logs verbose mode", + suggestions: [false, true] + }, + %{ + key: :prune, + type: [:atom, :tuple], + description: + "Non-retryable jobs [pruning settings](https://github.com/sorentwo/oban#pruning)", + suggestions: [:disabled, {:maxlen, 1500}, {:maxage, 60 * 60}] + }, + %{ + key: :queues, + type: :keyword, + description: + "Background jobs queues (keys: queues, values: max numbers of concurrent jobs)", + suggestions: [ + [ + activity_expiration: 10, + background: 5, + federator_incoming: 50, + federator_outgoing: 50, + mailer: 10, + scheduled_activities: 10, + transmogrifier: 20, + web_push: 50 + ] + ], + children: [ + %{ + key: :activity_expiration, + type: :integer, + description: "Activity expiration queue", + suggestions: [10] + }, + %{ + key: :background, + type: :integer, + description: "Background queue", + suggestions: [5] + }, + %{ + key: :federator_incoming, + type: :integer, + description: "Incoming federation queue", + suggestions: [50] + }, + %{ + key: :federator_outgoing, + type: :integer, + description: "Outgoing federation queue", + suggestions: [50] + }, + %{ + key: :mailer, + type: :integer, + description: "Email sender queue, see Pleroma.Emails.Mailer", + suggestions: [10] + }, + %{ + key: :scheduled_activities, + type: :integer, + description: "Scheduled activities queue, see Pleroma.ScheduledActivities", + suggestions: [10] + }, + %{ + key: :transmogrifier, + type: :integer, + description: "Transmogrifier queue", + suggestions: [20] + }, + %{ + key: :web_push, + type: :integer, + description: "Web push notifications queue", + suggestions: [50] + } + ] + } + ] + }, + %{ + group: :pleroma, + key: :workers, + type: :group, + description: "Includes custom worker options not interpretable directly by `Oban`", + children: [ + %{ + key: :retries, + type: :keyword, + description: "Max retry attempts for failed jobs, per `Oban` queue", + suggestions: [ + [ + federator_incoming: 5, + federator_outgoing: 5 + ] + ] } ] }, From 3b8ec98b0e3b3fb2bd333f3be724676c4821366f Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 15 Sep 2019 10:15:57 +0300 Subject: [PATCH 33/45] [#1149] Reinstated docs/config.md changes. --- docs/config.md | 1456 +++++++++++++++++++++++++----------------------- 1 file changed, 755 insertions(+), 701 deletions(-) diff --git a/docs/config.md b/docs/config.md index 066547bb1..270d7fcea 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1,701 +1,755 @@ -# Configuration - -This file describe the configuration, it is recommended to edit the relevant *.secret.exs file instead of the others founds in the ``config`` directory. -If you run Pleroma with ``MIX_ENV=prod`` the file is ``prod.secret.exs``, otherwise it is ``dev.secret.exs``. - -## Pleroma.Upload -* `uploader`: Select which `Pleroma.Uploaders` to use -* `filters`: List of `Pleroma.Upload.Filter` to use. -* `link_name`: When enabled Pleroma will add a `name` parameter to the url of the upload, for example `https://instance.tld/media/corndog.png?name=corndog.png`. This is needed to provide the correct filename in Content-Disposition headers when using filters like `Pleroma.Upload.Filter.Dedupe` -* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host. -* `proxy_remote`: If you're using a remote uploader, Pleroma will proxy media requests instead of redirecting to it. -* `proxy_opts`: Proxy options, see `Pleroma.ReverseProxy` documentation. - -Note: `strip_exif` has been replaced by `Pleroma.Upload.Filter.Mogrify`. - -## Pleroma.Uploaders.Local -* `uploads`: Which directory to store the user-uploads in, relative to pleroma’s working directory - -## Pleroma.Uploaders.S3 -* `bucket`: S3 bucket name -* `bucket_namespace`: S3 bucket namespace -* `public_endpoint`: S3 endpoint that the user finally accesses(ex. "https://s3.dualstack.ap-northeast-1.amazonaws.com") -* `truncated_namespace`: If you use S3 compatible service such as Digital Ocean Spaces or CDN, set folder name or "" etc. -For example, when using CDN to S3 virtual host format, set "". -At this time, write CNAME to CDN in public_endpoint. - -## Pleroma.Upload.Filter.Mogrify - -* `args`: List of actions for the `mogrify` command like `"strip"` or `["strip", "auto-orient", {"implode", "1"}]`. - -## Pleroma.Upload.Filter.Dedupe - -No specific configuration. - -## Pleroma.Upload.Filter.AnonymizeFilename - -This filter replaces the filename (not the path) of an upload. For complete obfuscation, add -`Pleroma.Upload.Filter.Dedupe` before AnonymizeFilename. - -* `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used. You can get the original filename extension by using `{extension}`, for example `custom-file-name.{extension}`. - -## Pleroma.Emails.Mailer -* `adapter`: one of the mail adapters listed in [Swoosh readme](https://github.com/swoosh/swoosh#adapters), or `Swoosh.Adapters.Local` for in-memory mailbox. -* `api_key` / `password` and / or other adapter-specific settings, per the above documentation. -* `enabled`: Allows enable/disable send emails. Default: `false`. - -An example for Sendgrid adapter: - -```elixir -config :pleroma, Pleroma.Emails.Mailer, - adapter: Swoosh.Adapters.Sendgrid, - api_key: "YOUR_API_KEY" -``` - -An example for SMTP adapter: - -```elixir -config :pleroma, Pleroma.Emails.Mailer, - adapter: Swoosh.Adapters.SMTP, - relay: "smtp.gmail.com", - username: "YOUR_USERNAME@gmail.com", - password: "YOUR_SMTP_PASSWORD", - port: 465, - ssl: true, - tls: :always, - auth: :always -``` - -## :uri_schemes -* `valid_schemes`: List of the scheme part that is considered valid to be an URL - -## :instance -* `name`: The instance’s name -* `email`: Email used to reach an Administrator/Moderator of the instance -* `notify_email`: Email used for notifications. -* `description`: The instance’s description, can be seen in nodeinfo and ``/api/v1/instance`` -* `limit`: Posts character limit (CW/Subject included in the counter) -* `remote_limit`: Hard character limit beyond which remote posts will be dropped. -* `upload_limit`: File size limit of uploads (except for avatar, background, banner) -* `avatar_upload_limit`: File size limit of user’s profile avatars -* `background_upload_limit`: File size limit of user’s profile backgrounds -* `banner_upload_limit`: File size limit of user’s profile banners -* `poll_limits`: A map with poll limits for **local** polls - * `max_options`: Maximum number of options - * `max_option_chars`: Maximum number of characters per option - * `min_expiration`: Minimum expiration time (in seconds) - * `max_expiration`: Maximum expiration time (in seconds) -* `registrations_open`: Enable registrations for anyone, invitations can be enabled when false. -* `invites_enabled`: Enable user invitations for admins (depends on `registrations_open: false`). -* `account_activation_required`: Require users to confirm their emails before signing in. -* `federating`: Enable federation with other instances -* `federation_incoming_replies_max_depth`: Max. depth of reply-to activities fetching on incoming federation, to prevent out-of-memory situations while fetching very long threads. If set to `nil`, threads of any depth will be fetched. Lower this value if you experience out-of-memory crashes. -* `federation_reachability_timeout_days`: Timeout (in days) of each external federation target being unreachable prior to pausing federating to it. -* `allow_relay`: Enable Pleroma’s Relay, which makes it possible to follow a whole instance -* `rewrite_policy`: Message Rewrite Policy, either one or a list. Here are the ones available by default: - * `Pleroma.Web.ActivityPub.MRF.NoOpPolicy`: Doesn’t modify activities (default) - * `Pleroma.Web.ActivityPub.MRF.DropPolicy`: Drops all activities. It generally doesn’t makes sense to use in production - * `Pleroma.Web.ActivityPub.MRF.SimplePolicy`: Restrict the visibility of activities from certains instances (See ``:mrf_simple`` section) - * `Pleroma.Web.ActivityPub.MRF.TagPolicy`: Applies policies to individual users based on tags, which can be set using pleroma-fe/admin-fe/any other app that supports Pleroma Admin API. For example it allows marking posts from individual users nsfw (sensitive) - * `Pleroma.Web.ActivityPub.MRF.SubchainPolicy`: Selectively runs other MRF policies when messages match (see ``:mrf_subchain`` section) - * `Pleroma.Web.ActivityPub.MRF.RejectNonPublic`: Drops posts with non-public visibility settings (See ``:mrf_rejectnonpublic`` section) - * `Pleroma.Web.ActivityPub.MRF.EnsureRePrepended`: Rewrites posts to ensure that replies to posts with subjects do not have an identical subject and instead begin with re:. - * `Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy`: Rejects posts from likely spambots by rejecting posts from new users that contain links. - * `Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`: Crawls attachments using their MediaProxy URLs so that the MediaProxy cache is primed. - * `Pleroma.Web.ActivityPub.MRF.MentionPolicy`: Drops posts mentioning configurable users. (see `:mrf_mention` section) - * `Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`: Restricts activities to a configured set of vocabulary. (see `:mrf_vocabulary` section) -* `public`: Makes the client API in authentificated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. -* `quarantined_instances`: List of ActivityPub instances where private(DMs, followers-only) activities will not be send. -* `managed_config`: Whenether the config for pleroma-fe is configured in this config or in ``static/config.json`` -* `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML) -* `mrf_transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo). -* `mrf_transparency_exclusions`: Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value. -* `scope_copy`: Copy the scope (private/unlisted/public) in replies to posts by default. -* `subject_line_behavior`: Allows changing the default behaviour of subject lines in replies. Valid values: - * "email": Copy and preprend re:, as in email. - * "masto": Copy verbatim, as in Mastodon. - * "noop": Don't copy the subject. -* `always_show_subject_input`: When set to false, auto-hide the subject field when it's empty. -* `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with - older software for theses nicknames. -* `max_pinned_statuses`: The maximum number of pinned statuses. `0` will disable the feature. -* `autofollowed_nicknames`: Set to nicknames of (local) users that every new user should automatically follow. -* `no_attachment_links`: Set to true to disable automatically adding attachment link text to statuses -* `welcome_message`: A message that will be send to a newly registered users as a direct message. -* `welcome_user_nickname`: The nickname of the local user that sends the welcome message. -* `max_report_comment_size`: The maximum size of the report comment (Default: `1000`) -* `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). Default: `false`. -* `healthcheck`: If set to true, system data will be shown on ``/api/pleroma/healthcheck``. -* `remote_post_retention_days`: The default amount of days to retain remote posts when pruning the database. -* `user_bio_length`: A user bio maximum length (default: `5000`) -* `user_name_length`: A user name maximum length (default: `100`) -* `skip_thread_containment`: Skip filter out broken threads. The default is `false`. -* `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. -* `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. -* `max_account_fields`: The maximum number of custom fields in the user profile (default: `10`) -* `max_remote_account_fields`: The maximum number of custom fields in the remote user profile (default: `20`) -* `account_field_name_length`: An account field name maximum length (default: `512`) -* `account_field_value_length`: An account field value maximum length (default: `512`) -* `external_user_synchronization`: Enabling following/followers counters synchronization for external users. - - - -## :logger -* `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog, and `Quack.Logger` to log to Slack - -An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed: -```elixir -config :logger, - backends: [{ExSyslogger, :ex_syslogger}] - -config :logger, :ex_syslogger, - level: :warn -``` - -Another example, keeping console output and adding the pid to syslog output: -```elixir -config :logger, - backends: [:console, {ExSyslogger, :ex_syslogger}] - -config :logger, :ex_syslogger, - level: :warn, - option: [:pid, :ndelay] -``` - -See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_syslogger’s documentation](https://hexdocs.pm/ex_syslogger/) - -An example of logging info to local syslog, but warn to a Slack channel: -```elixir -config :logger, - backends: [ {ExSyslogger, :ex_syslogger}, Quack.Logger ], - level: :info - -config :logger, :ex_syslogger, - level: :info, - ident: "pleroma", - format: "$metadata[$level] $message" - -config :quack, - level: :warn, - meta: [:all], - webhook_url: "https://hooks.slack.com/services/YOUR-API-KEY-HERE" -``` - -See the [Quack Github](https://github.com/azohra/quack) for more details - -## :frontend_configurations - -This can be used to configure a keyword list that keeps the configuration data for any kind of frontend. By default, settings for `pleroma_fe` and `masto_fe` are configured. - -Frontends can access these settings at `/api/pleroma/frontend_configurations` - -To add your own configuration for PleromaFE, use it like this: - -```elixir -config :pleroma, :frontend_configurations, - pleroma_fe: %{ - theme: "pleroma-dark", - # ... see /priv/static/static/config.json for the available keys. -}, - masto_fe: %{ - showInstanceSpecificPanel: true - } -``` - -These settings **need to be complete**, they will override the defaults. - -NOTE: for versions < 1.0, you need to set [`:fe`](#fe) to false, as shown a few lines below. - -## :fe -__THIS IS DEPRECATED__ - -If you are using this method, please change it to the [`frontend_configurations`](#frontend_configurations) method. -Please **set this option to false** in your config like this: - -```elixir -config :pleroma, :fe, false -``` - -This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:instance`` is set to false. - -* `theme`: Which theme to use, they are defined in ``styles.json`` -* `logo`: URL of the logo, defaults to Pleroma’s logo -* `logo_mask`: Whether to use only the logo's shape as a mask (true) or as a regular image (false) -* `logo_margin`: What margin to use around the logo -* `background`: URL of the background, unless viewing a user profile with a background that is set -* `redirect_root_no_login`: relative URL which indicates where to redirect when a user isn’t logged in. -* `redirect_root_login`: relative URL which indicates where to redirect when a user is logged in. -* `show_instance_panel`: Whenether to show the instance’s specific panel. -* `scope_options_enabled`: Enable setting an notice visibility and subject/CW when posting -* `formatting_options_enabled`: Enable setting a formatting different than plain-text (ie. HTML, Markdown) when posting, relates to ``:instance, allowed_post_formats`` -* `collapse_message_with_subjects`: When a message has a subject(aka Content Warning), collapse it by default -* `hide_post_stats`: Hide notices statistics(repeats, favorites, …) -* `hide_user_stats`: Hide profile statistics(posts, posts per day, followers, followings, …) - -## :assets - -This section configures assets to be used with various frontends. Currently the only option -relates to mascots on the mastodon frontend - -* `mascots`: KeywordList of mascots, each element __MUST__ contain both a `url` and a - `mime_type` key. -* `default_mascot`: An element from `mascots` - This will be used as the default mascot - on MastoFE (default: `:pleroma_fox_tan`) - -## :mrf_simple -* `media_removal`: List of instances to remove medias from -* `media_nsfw`: List of instances to put medias as NSFW(sensitive) from -* `federated_timeline_removal`: List of instances to remove from Federated (aka The Whole Known Network) Timeline -* `reject`: List of instances to reject any activities from -* `accept`: List of instances to accept any activities from -* `report_removal`: List of instances to reject reports from -* `avatar_removal`: List of instances to strip avatars from -* `banner_removal`: List of instances to strip banners from - -## :mrf_subchain -This policy processes messages through an alternate pipeline when a given message matches certain criteria. -All criteria are configured as a map of regular expressions to lists of policy modules. - -* `match_actor`: Matches a series of regular expressions against the actor field. - -Example: - -``` -config :pleroma, :mrf_subchain, - match_actor: %{ - ~r/https:\/\/example.com/s => [Pleroma.Web.ActivityPub.MRF.DropPolicy] - } -``` - -## :mrf_rejectnonpublic -* `allow_followersonly`: whether to allow followers-only posts -* `allow_direct`: whether to allow direct messages - -## :mrf_hellthread -* `delist_threshold`: Number of mentioned users after which the message gets delisted (the message can still be seen, but it will not show up in public timelines and mentioned users won't get notifications about it). Set to 0 to disable. -* `reject_threshold`: Number of mentioned users after which the messaged gets rejected. Set to 0 to disable. - -## :mrf_keyword -* `reject`: A list of patterns which result in message being rejected, each pattern can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) -* `federated_timeline_removal`: A list of patterns which result in message being removed from federated timelines (a.k.a unlisted), each pattern can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) -* `replace`: A list of tuples containing `{pattern, replacement}`, `pattern` can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) - -## :mrf_mention -* `actors`: A list of actors, for which to drop any posts mentioning. - -## :mrf_vocabulary -* `accept`: A list of ActivityStreams terms to accept. If empty, all supported messages are accepted. -* `reject`: A list of ActivityStreams terms to reject. If empty, no messages are rejected. - -## :media_proxy -* `enabled`: Enables proxying of remote media to the instance’s proxy -* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts. -* `proxy_opts`: All options defined in `Pleroma.ReverseProxy` documentation, defaults to `[max_body_length: (25*1_048_576)]`. -* `whitelist`: List of domains to bypass the mediaproxy - -## :gopher -* `enabled`: Enables the gopher interface -* `ip`: IP address to bind to -* `port`: Port to bind to -* `dstport`: Port advertised in urls (optional, defaults to `port`) - -## Pleroma.Web.Endpoint -`Phoenix` endpoint configuration, all configuration options can be viewed [here](https://hexdocs.pm/phoenix/Phoenix.Endpoint.html#module-dynamic-configuration), only common options are listed here -* `http` - a list containing http protocol configuration, all configuration options can be viewed [here](https://hexdocs.pm/plug_cowboy/Plug.Cowboy.html#module-options), only common options are listed here. For deployment using docker, you need to set this to `[ip: {0,0,0,0}, port: 4000]` to make pleroma accessible from other containers (such as your nginx server). - - `ip` - a tuple consisting of 4 integers - - `port` -* `url` - a list containing the configuration for generating urls, accepts - - `host` - the host without the scheme and a post (e.g `example.com`, not `https://example.com:2020`) - - `scheme` - e.g `http`, `https` - - `port` - - `path` -* `extra_cookie_attrs` - a list of `Key=Value` strings to be added as non-standard cookie attributes. Defaults to `["SameSite=Lax"]`. See the [SameSite article](https://www.owasp.org/index.php/SameSite) on OWASP for more info. - - - -**Important note**: if you modify anything inside these lists, default `config.exs` values will be overwritten, which may result in breakage, to make sure this does not happen please copy the default value for the list from `config.exs` and modify/add only what you need - -Example: -```elixir -config :pleroma, Pleroma.Web.Endpoint, - url: [host: "example.com", port: 2020, scheme: "https"], - http: [ - # start copied from config.exs - dispatch: [ - {:_, - [ - {"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []}, - {"/websocket", Phoenix.Endpoint.CowboyWebSocket, - {Phoenix.Transports.WebSocket, - {Pleroma.Web.Endpoint, Pleroma.Web.UserSocket, websocket_config}}}, - {:_, Phoenix.Endpoint.Cowboy2Handler, {Pleroma.Web.Endpoint, []}} - ]} - # end copied from config.exs - ], - port: 8080, - ip: {127, 0, 0, 1} - ] -``` - -This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls starting with `https://example.com:2020` - -## :activitypub -* ``unfollow_blocked``: Whether blocks result in people getting unfollowed -* ``outgoing_blocks``: Whether to federate blocks to other instances -* ``deny_follow_blocked``: Whether to disallow following an account that has blocked the user in question -* ``sign_object_fetches``: Sign object fetches with HTTP signatures - -## :http_security -* ``enabled``: Whether the managed content security policy is enabled -* ``sts``: Whether to additionally send a `Strict-Transport-Security` header -* ``sts_max_age``: The maximum age for the `Strict-Transport-Security` header if sent -* ``ct_max_age``: The maximum age for the `Expect-CT` header if sent -* ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"` -* ``report_uri``: Adds the specified url to `report-uri` and `report-to` group in CSP header. - -## :mrf_user_allowlist - -The keys in this section are the domain names that the policy should apply to. -Each key should be assigned a list of users that should be allowed through by -their ActivityPub ID. - -An example: - -```elixir -config :pleroma, :mrf_user_allowlist, - "example.org": ["https://example.org/users/admin"] -``` - -## :web_push_encryption, :vapid_details - -Web Push Notifications configuration. You can use the mix task `mix web_push.gen.keypair` to generate it. - -* ``subject``: a mailto link for the administrative contact. It’s best if this email is not a personal email address, but rather a group email so that if a person leaves an organization, is unavailable for an extended period, or otherwise can’t respond, someone else on the list can. -* ``public_key``: VAPID public key -* ``private_key``: VAPID private key - -## Pleroma.Captcha -* `enabled`: Whether the captcha should be shown on registration -* `method`: The method/service to use for captcha -* `seconds_valid`: The time in seconds for which the captcha is valid - -### Pleroma.Captcha.Kocaptcha -Kocaptcha is a very simple captcha service with a single API endpoint, -the source code is here: https://github.com/koto-bank/kocaptcha. The default endpoint -`https://captcha.kotobank.ch` is hosted by the developer. - -* `endpoint`: the kocaptcha endpoint to use - -## :admin_token - -Allows to set a token that can be used to authenticate with the admin api without using an actual user by giving it as the 'admin_token' parameter. Example: - -```elixir -config :pleroma, :admin_token, "somerandomtoken" -``` - -You can then do - -```sh -curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerandomtoken" -``` - -## :pleroma_job_queue - -[Pleroma Job Queue](https://git.pleroma.social/pleroma/pleroma_job_queue) configuration: a list of queues with maximum concurrent jobs. - -Pleroma has the following queues: - -* `federator_outgoing` - Outgoing federation -* `federator_incoming` - Incoming federation -* `mailer` - Email sender, see [`Pleroma.Emails.Mailer`](#pleroma-emails-mailer) -* `transmogrifier` - Transmogrifier -* `web_push` - Web push notifications -* `scheduled_activities` - Scheduled activities, see [`Pleroma.ScheduledActivities`](#pleromascheduledactivity) - -Example: - -```elixir -config :pleroma_job_queue, :queues, - federator_incoming: 50, - federator_outgoing: 50 -``` - -This config contains two queues: `federator_incoming` and `federator_outgoing`. Both have the `max_jobs` set to `50`. - -## Pleroma.Web.Federator.RetryQueue - -* `enabled`: If set to `true`, failed federation jobs will be retried -* `max_jobs`: The maximum amount of parallel federation jobs running at the same time. -* `initial_timeout`: The initial timeout in seconds -* `max_retries`: The maximum number of times a federation job is retried - -## Pleroma.Web.Metadata -* `providers`: a list of metadata providers to enable. Providers available: - * Pleroma.Web.Metadata.Providers.OpenGraph - * Pleroma.Web.Metadata.Providers.TwitterCard - * Pleroma.Web.Metadata.Providers.RelMe - add links from user bio with rel=me into the `
` as `` -* `unfurl_nsfw`: If set to `true` nsfw attachments will be shown in previews - -## :rich_media -* `enabled`: if enabled the instance will parse metadata from attached links to generate link previews -* `ignore_hosts`: list of hosts which will be ignored by the metadata parser. For example `["accounts.google.com", "xss.website"]`, defaults to `[]`. -* `ignore_tld`: list TLDs (top-level domains) which will ignore for parse metadata. default is ["local", "localdomain", "lan"] -* `parsers`: list of Rich Media parsers - -## :fetch_initial_posts -* `enabled`: if enabled, when a new user is federated with, fetch some of their latest posts -* `pages`: the amount of pages to fetch - -## :hackney_pools - -Advanced. Tweaks Hackney (http client) connections pools. - -There's three pools used: - -* `:federation` for the federation jobs. - You may want this pool max_connections to be at least equal to the number of federator jobs + retry queue jobs. -* `:media` for rich media, media proxy -* `:upload` for uploaded media (if using a remote uploader and `proxy_remote: true`) - -For each pool, the options are: - -* `max_connections` - how much connections a pool can hold -* `timeout` - retention duration for connections - -## :auto_linker - -Configuration for the `auto_linker` library: - -* `class: "auto-linker"` - specify the class to be added to the generated link. false to clear -* `rel: "noopener noreferrer"` - override the rel attribute. false to clear -* `new_window: true` - set to false to remove `target='_blank'` attribute -* `scheme: false` - Set to true to link urls with schema `http://google.com` -* `truncate: false` - Set to a number to truncate urls longer then the number. Truncated urls will end in `..` -* `strip_prefix: true` - Strip the scheme prefix -* `extra: false` - link urls with rarely used schemes (magnet, ipfs, irc, etc.) - -Example: - -```elixir -config :auto_linker, - opts: [ - scheme: true, - extra: true, - class: false, - strip_prefix: false, - new_window: false, - rel: false - ] -``` - -## Pleroma.ScheduledActivity - -* `daily_user_limit`: the number of scheduled activities a user is allowed to create in a single day (Default: `25`) -* `total_user_limit`: the number of scheduled activities a user is allowed to create in total (Default: `300`) -* `enabled`: whether scheduled activities are sent to the job queue to be executed - -## Pleroma.ActivityExpiration - -# `enabled`: whether expired activities will be sent to the job queue to be deleted - -## Pleroma.Web.Auth.Authenticator - -* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator -* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication - -## :ldap - -Use LDAP for user authentication. When a user logs in to the Pleroma -instance, the name and password will be verified by trying to authenticate -(bind) to an LDAP server. If a user exists in the LDAP directory but there -is no account with the same name yet on the Pleroma instance then a new -Pleroma account will be created with the same name as the LDAP user name. - -* `enabled`: enables LDAP authentication -* `host`: LDAP server hostname -* `port`: LDAP port, e.g. 389 or 636 -* `ssl`: true to use SSL, usually implies the port 636 -* `sslopts`: additional SSL options -* `tls`: true to start TLS, usually implies the port 389 -* `tlsopts`: additional TLS options -* `base`: LDAP base, e.g. "dc=example,dc=com" -* `uid`: LDAP attribute name to authenticate the user, e.g. when "cn", the filter will be "cn=username,base" - -## BBS / SSH access - -To enable simple command line interface accessible over ssh, add a setting like this to your configuration file: - -```exs -app_dir = File.cwd! -priv_dir = Path.join([app_dir, "priv/ssh_keys"]) - -config :esshd, - enabled: true, - priv_dir: priv_dir, - handler: "Pleroma.BBS.Handler", - port: 10_022, - password_authenticator: "Pleroma.BBS.Authenticator" -``` - -Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -m PEM -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT` - -## :auth - -* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator -* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication - -Authentication / authorization settings. - -* `auth_template`: authentication form template. By default it's `show.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/show.html.eex`. -* `oauth_consumer_template`: OAuth consumer mode authentication form template. By default it's `consumer.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex`. -* `oauth_consumer_strategies`: the list of enabled OAuth consumer strategies; by default it's set by `OAUTH_CONSUMER_STRATEGIES` environment variable. Each entry in this space-delimited string should be of format `` or `:` (e.g. `twitter` or `keycloak:ueberauth_keycloak_strategy` in case dependency is named differently than `ueberauth_`). - -## :email_notifications - -Email notifications settings. - - - digest - emails of "what you've missed" for users who have been - inactive for a while. - - active: globally enable or disable digest emails - - schedule: When to send digest email, in [crontab format](https://en.wikipedia.org/wiki/Cron). - "0 0 * * 0" is the default, meaning "once a week at midnight on Sunday morning" - - interval: Minimum interval between digest emails to one user - - inactivity_threshold: Minimum user inactivity threshold - -## Pleroma.Emails.UserEmail - -- `:logo` - a path to a custom logo. Set it to `nil` to use the default Pleroma logo. -- `:styling` - a map with color settings for email templates. - -## OAuth consumer mode - -OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.). -Implementation is based on Ueberauth; see the list of [available strategies](https://github.com/ueberauth/ueberauth/wiki/List-of-Strategies). - -Note: each strategy is shipped as a separate dependency; in order to get the strategies, run `OAUTH_CONSUMER_STRATEGIES="..." mix deps.get`, -e.g. `OAUTH_CONSUMER_STRATEGIES="twitter facebook google microsoft" mix deps.get`. -The server should also be started with `OAUTH_CONSUMER_STRATEGIES="..." mix phx.server` in case you enable any strategies. - -Note: each strategy requires separate setup (on external provider side and Pleroma side). Below are the guidelines on setting up most popular strategies. - -Note: make sure that `"SameSite=Lax"` is set in `extra_cookie_attrs` when you have this feature enabled. OAuth consumer mode will not work with `"SameSite=Strict"` - -* For Twitter, [register an app](https://developer.twitter.com/en/apps), configure callback URL to https:///oauth/twitter/callback - -* For Facebook, [register an app](https://developers.facebook.com/apps), configure callback URL to https:///oauth/facebook/callback, enable Facebook Login service at https://developers.facebook.com/apps//fb-login/settings/ - -* For Google, [register an app](https://console.developers.google.com), configure callback URL to https:///oauth/google/callback - -* For Microsoft, [register an app](https://portal.azure.com), configure callback URL to https:///oauth/microsoft/callback - -Once the app is configured on external OAuth provider side, add app's credentials and strategy-specific settings (if any — e.g. see Microsoft below) to `config/prod.secret.exs`, -per strategy's documentation (e.g. [ueberauth_twitter](https://github.com/ueberauth/ueberauth_twitter)). Example config basing on environment variables: - -```elixir -# Twitter -config :ueberauth, Ueberauth.Strategy.Twitter.OAuth, - consumer_key: System.get_env("TWITTER_CONSUMER_KEY"), - consumer_secret: System.get_env("TWITTER_CONSUMER_SECRET") - -# Facebook -config :ueberauth, Ueberauth.Strategy.Facebook.OAuth, - client_id: System.get_env("FACEBOOK_APP_ID"), - client_secret: System.get_env("FACEBOOK_APP_SECRET"), - redirect_uri: System.get_env("FACEBOOK_REDIRECT_URI") - -# Google -config :ueberauth, Ueberauth.Strategy.Google.OAuth, - client_id: System.get_env("GOOGLE_CLIENT_ID"), - client_secret: System.get_env("GOOGLE_CLIENT_SECRET"), - redirect_uri: System.get_env("GOOGLE_REDIRECT_URI") - -# Microsoft -config :ueberauth, Ueberauth.Strategy.Microsoft.OAuth, - client_id: System.get_env("MICROSOFT_CLIENT_ID"), - client_secret: System.get_env("MICROSOFT_CLIENT_SECRET") - -config :ueberauth, Ueberauth, - providers: [ - microsoft: {Ueberauth.Strategy.Microsoft, [callback_params: []]} - ] - -# Keycloak -# Note: make sure to add `keycloak:ueberauth_keycloak_strategy` entry to `OAUTH_CONSUMER_STRATEGIES` environment variable -keycloak_url = "https://publicly-reachable-keycloak-instance.org:8080" - -config :ueberauth, Ueberauth.Strategy.Keycloak.OAuth, - client_id: System.get_env("KEYCLOAK_CLIENT_ID"), - client_secret: System.get_env("KEYCLOAK_CLIENT_SECRET"), - site: keycloak_url, - authorize_url: "#{keycloak_url}/auth/realms/master/protocol/openid-connect/auth", - token_url: "#{keycloak_url}/auth/realms/master/protocol/openid-connect/token", - userinfo_url: "#{keycloak_url}/auth/realms/master/protocol/openid-connect/userinfo", - token_method: :post - -config :ueberauth, Ueberauth, - providers: [ - keycloak: {Ueberauth.Strategy.Keycloak, [uid_field: :email]} - ] -``` - -## OAuth 2.0 provider - :oauth2 - -Configure OAuth 2 provider capabilities: - -* `token_expires_in` - The lifetime in seconds of the access token. -* `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token. -* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`. -* `clean_expired_tokens_interval` - Interval to run the job to clean expired tokens. Defaults to `86_400_000` (24 hours). - -## :emoji -* `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]` -* `pack_extensions`: A list of file extensions for emojis, when no emoji.txt for a pack is present. Example `[".png", ".gif"]` -* `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]` -* `default_manifest`: Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download. Currently only one manifest can be added (no arrays). - -## Database options - -### RUM indexing for full text search -* `rum_enabled`: If RUM indexes should be used. Defaults to `false`. - -RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. While they may eventually be mainlined, for now they have to be installed as a PostgreSQL extension from https://github.com/postgrespro/rum. - -Their advantage over the standard GIN indexes is that they allow efficient ordering of search results by timestamp, which makes search queries a lot faster on larger servers, by one or two orders of magnitude. They take up around 3 times as much space as GIN indexes. - -To enable them, both the `rum_enabled` flag has to be set and the following special migration has to be run: - -`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/` - -This will probably take a long time. - -## :rate_limit - -This is an advanced feature and disabled by default. - -A keyword list of rate limiters where a key is a limiter name and value is the limiter configuration. The basic configuration is a tuple where: - -* The first element: `scale` (Integer). The time scale in milliseconds. -* The second element: `limit` (Integer). How many requests to limit in the time scale provided. - -It is also possible to have different limits for unauthenticated and authenticated users: the keyword value must be a list of two tuples where the first one is a config for unauthenticated users and the second one is for authenticated. - -See [`Pleroma.Plugs.RateLimiter`](Pleroma.Plugs.RateLimiter.html) documentation for examples. - -Supported rate limiters: - -* `:search` for the search requests (account & status search etc.) -* `:app_account_creation` for registering user accounts from the same IP address -* `:relations_actions` for actions on relations with all users (follow, unfollow) -* `:relation_id_action` for actions on relation with a specific user (follow, unfollow) -* `:statuses_actions` for create / delete / fav / unfav / reblog / unreblog actions on any statuses -* `:status_id_action` for fav / unfav or reblog / unreblog actions on the same status by the same user - -## :web_cache_ttl - -The expiration time for the web responses cache. Values should be in milliseconds or `nil` to disable expiration. - -Available caches: - -* `:activity_pub` - activity pub routes (except question activities). Defaults to `nil` (no expiration). -* `:activity_pub_question` - activity pub routes (question activities). Defaults to `30_000` (30 seconds). +# Configuration + +This file describe the configuration, it is recommended to edit the relevant *.secret.exs file instead of the others founds in the ``config`` directory. +If you run Pleroma with ``MIX_ENV=prod`` the file is ``prod.secret.exs``, otherwise it is ``dev.secret.exs``. + +## Pleroma.Upload +* `uploader`: Select which `Pleroma.Uploaders` to use +* `filters`: List of `Pleroma.Upload.Filter` to use. +* `link_name`: When enabled Pleroma will add a `name` parameter to the url of the upload, for example `https://instance.tld/media/corndog.png?name=corndog.png`. This is needed to provide the correct filename in Content-Disposition headers when using filters like `Pleroma.Upload.Filter.Dedupe` +* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host. +* `proxy_remote`: If you're using a remote uploader, Pleroma will proxy media requests instead of redirecting to it. +* `proxy_opts`: Proxy options, see `Pleroma.ReverseProxy` documentation. + +Note: `strip_exif` has been replaced by `Pleroma.Upload.Filter.Mogrify`. + +## Pleroma.Uploaders.Local +* `uploads`: Which directory to store the user-uploads in, relative to pleroma’s working directory + +## Pleroma.Uploaders.S3 +* `bucket`: S3 bucket name +* `bucket_namespace`: S3 bucket namespace +* `public_endpoint`: S3 endpoint that the user finally accesses(ex. "https://s3.dualstack.ap-northeast-1.amazonaws.com") +* `truncated_namespace`: If you use S3 compatible service such as Digital Ocean Spaces or CDN, set folder name or "" etc. +For example, when using CDN to S3 virtual host format, set "". +At this time, write CNAME to CDN in public_endpoint. + +## Pleroma.Upload.Filter.Mogrify + +* `args`: List of actions for the `mogrify` command like `"strip"` or `["strip", "auto-orient", {"implode", "1"}]`. + +## Pleroma.Upload.Filter.Dedupe + +No specific configuration. + +## Pleroma.Upload.Filter.AnonymizeFilename + +This filter replaces the filename (not the path) of an upload. For complete obfuscation, add +`Pleroma.Upload.Filter.Dedupe` before AnonymizeFilename. + +* `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used. You can get the original filename extension by using `{extension}`, for example `custom-file-name.{extension}`. + +## Pleroma.Emails.Mailer +* `adapter`: one of the mail adapters listed in [Swoosh readme](https://github.com/swoosh/swoosh#adapters), or `Swoosh.Adapters.Local` for in-memory mailbox. +* `api_key` / `password` and / or other adapter-specific settings, per the above documentation. +* `enabled`: Allows enable/disable send emails. Default: `false`. + +An example for Sendgrid adapter: + +```elixir +config :pleroma, Pleroma.Emails.Mailer, + adapter: Swoosh.Adapters.Sendgrid, + api_key: "YOUR_API_KEY" +``` + +An example for SMTP adapter: + +```elixir +config :pleroma, Pleroma.Emails.Mailer, + adapter: Swoosh.Adapters.SMTP, + relay: "smtp.gmail.com", + username: "YOUR_USERNAME@gmail.com", + password: "YOUR_SMTP_PASSWORD", + port: 465, + ssl: true, + tls: :always, + auth: :always +``` + +## :uri_schemes +* `valid_schemes`: List of the scheme part that is considered valid to be an URL + +## :instance +* `name`: The instance’s name +* `email`: Email used to reach an Administrator/Moderator of the instance +* `notify_email`: Email used for notifications. +* `description`: The instance’s description, can be seen in nodeinfo and ``/api/v1/instance`` +* `limit`: Posts character limit (CW/Subject included in the counter) +* `remote_limit`: Hard character limit beyond which remote posts will be dropped. +* `upload_limit`: File size limit of uploads (except for avatar, background, banner) +* `avatar_upload_limit`: File size limit of user’s profile avatars +* `background_upload_limit`: File size limit of user’s profile backgrounds +* `banner_upload_limit`: File size limit of user’s profile banners +* `poll_limits`: A map with poll limits for **local** polls + * `max_options`: Maximum number of options + * `max_option_chars`: Maximum number of characters per option + * `min_expiration`: Minimum expiration time (in seconds) + * `max_expiration`: Maximum expiration time (in seconds) +* `registrations_open`: Enable registrations for anyone, invitations can be enabled when false. +* `invites_enabled`: Enable user invitations for admins (depends on `registrations_open: false`). +* `account_activation_required`: Require users to confirm their emails before signing in. +* `federating`: Enable federation with other instances +* `federation_incoming_replies_max_depth`: Max. depth of reply-to activities fetching on incoming federation, to prevent out-of-memory situations while fetching very long threads. If set to `nil`, threads of any depth will be fetched. Lower this value if you experience out-of-memory crashes. +* `federation_reachability_timeout_days`: Timeout (in days) of each external federation target being unreachable prior to pausing federating to it. +* `allow_relay`: Enable Pleroma’s Relay, which makes it possible to follow a whole instance +* `rewrite_policy`: Message Rewrite Policy, either one or a list. Here are the ones available by default: + * `Pleroma.Web.ActivityPub.MRF.NoOpPolicy`: Doesn’t modify activities (default) + * `Pleroma.Web.ActivityPub.MRF.DropPolicy`: Drops all activities. It generally doesn’t makes sense to use in production + * `Pleroma.Web.ActivityPub.MRF.SimplePolicy`: Restrict the visibility of activities from certains instances (See ``:mrf_simple`` section) + * `Pleroma.Web.ActivityPub.MRF.TagPolicy`: Applies policies to individual users based on tags, which can be set using pleroma-fe/admin-fe/any other app that supports Pleroma Admin API. For example it allows marking posts from individual users nsfw (sensitive) + * `Pleroma.Web.ActivityPub.MRF.SubchainPolicy`: Selectively runs other MRF policies when messages match (see ``:mrf_subchain`` section) + * `Pleroma.Web.ActivityPub.MRF.RejectNonPublic`: Drops posts with non-public visibility settings (See ``:mrf_rejectnonpublic`` section) + * `Pleroma.Web.ActivityPub.MRF.EnsureRePrepended`: Rewrites posts to ensure that replies to posts with subjects do not have an identical subject and instead begin with re:. + * `Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy`: Rejects posts from likely spambots by rejecting posts from new users that contain links. + * `Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`: Crawls attachments using their MediaProxy URLs so that the MediaProxy cache is primed. + * `Pleroma.Web.ActivityPub.MRF.MentionPolicy`: Drops posts mentioning configurable users. (see `:mrf_mention` section) + * `Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`: Restricts activities to a configured set of vocabulary. (see `:mrf_vocabulary` section) +* `public`: Makes the client API in authentificated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. +* `quarantined_instances`: List of ActivityPub instances where private(DMs, followers-only) activities will not be send. +* `managed_config`: Whenether the config for pleroma-fe is configured in this config or in ``static/config.json`` +* `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML) +* `mrf_transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo). +* `mrf_transparency_exclusions`: Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value. +* `scope_copy`: Copy the scope (private/unlisted/public) in replies to posts by default. +* `subject_line_behavior`: Allows changing the default behaviour of subject lines in replies. Valid values: + * "email": Copy and preprend re:, as in email. + * "masto": Copy verbatim, as in Mastodon. + * "noop": Don't copy the subject. +* `always_show_subject_input`: When set to false, auto-hide the subject field when it's empty. +* `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with + older software for theses nicknames. +* `max_pinned_statuses`: The maximum number of pinned statuses. `0` will disable the feature. +* `autofollowed_nicknames`: Set to nicknames of (local) users that every new user should automatically follow. +* `no_attachment_links`: Set to true to disable automatically adding attachment link text to statuses +* `welcome_message`: A message that will be send to a newly registered users as a direct message. +* `welcome_user_nickname`: The nickname of the local user that sends the welcome message. +* `max_report_comment_size`: The maximum size of the report comment (Default: `1000`) +* `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). Default: `false`. +* `healthcheck`: If set to true, system data will be shown on ``/api/pleroma/healthcheck``. +* `remote_post_retention_days`: The default amount of days to retain remote posts when pruning the database. +* `user_bio_length`: A user bio maximum length (default: `5000`) +* `user_name_length`: A user name maximum length (default: `100`) +* `skip_thread_containment`: Skip filter out broken threads. The default is `false`. +* `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. +* `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. +* `max_account_fields`: The maximum number of custom fields in the user profile (default: `10`) +* `max_remote_account_fields`: The maximum number of custom fields in the remote user profile (default: `20`) +* `account_field_name_length`: An account field name maximum length (default: `512`) +* `account_field_value_length`: An account field value maximum length (default: `512`) +* `external_user_synchronization`: Enabling following/followers counters synchronization for external users. + + + +## :logger +* `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog, and `Quack.Logger` to log to Slack + +An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed: +```elixir +config :logger, + backends: [{ExSyslogger, :ex_syslogger}] + +config :logger, :ex_syslogger, + level: :warn +``` + +Another example, keeping console output and adding the pid to syslog output: +```elixir +config :logger, + backends: [:console, {ExSyslogger, :ex_syslogger}] + +config :logger, :ex_syslogger, + level: :warn, + option: [:pid, :ndelay] +``` + +See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_syslogger’s documentation](https://hexdocs.pm/ex_syslogger/) + +An example of logging info to local syslog, but warn to a Slack channel: +```elixir +config :logger, + backends: [ {ExSyslogger, :ex_syslogger}, Quack.Logger ], + level: :info + +config :logger, :ex_syslogger, + level: :info, + ident: "pleroma", + format: "$metadata[$level] $message" + +config :quack, + level: :warn, + meta: [:all], + webhook_url: "https://hooks.slack.com/services/YOUR-API-KEY-HERE" +``` + +See the [Quack Github](https://github.com/azohra/quack) for more details + +## :frontend_configurations + +This can be used to configure a keyword list that keeps the configuration data for any kind of frontend. By default, settings for `pleroma_fe` and `masto_fe` are configured. + +Frontends can access these settings at `/api/pleroma/frontend_configurations` + +To add your own configuration for PleromaFE, use it like this: + +```elixir +config :pleroma, :frontend_configurations, + pleroma_fe: %{ + theme: "pleroma-dark", + # ... see /priv/static/static/config.json for the available keys. +}, + masto_fe: %{ + showInstanceSpecificPanel: true + } +``` + +These settings **need to be complete**, they will override the defaults. + +NOTE: for versions < 1.0, you need to set [`:fe`](#fe) to false, as shown a few lines below. + +## :fe +__THIS IS DEPRECATED__ + +If you are using this method, please change it to the [`frontend_configurations`](#frontend_configurations) method. +Please **set this option to false** in your config like this: + +```elixir +config :pleroma, :fe, false +``` + +This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:instance`` is set to false. + +* `theme`: Which theme to use, they are defined in ``styles.json`` +* `logo`: URL of the logo, defaults to Pleroma’s logo +* `logo_mask`: Whether to use only the logo's shape as a mask (true) or as a regular image (false) +* `logo_margin`: What margin to use around the logo +* `background`: URL of the background, unless viewing a user profile with a background that is set +* `redirect_root_no_login`: relative URL which indicates where to redirect when a user isn’t logged in. +* `redirect_root_login`: relative URL which indicates where to redirect when a user is logged in. +* `show_instance_panel`: Whenether to show the instance’s specific panel. +* `scope_options_enabled`: Enable setting an notice visibility and subject/CW when posting +* `formatting_options_enabled`: Enable setting a formatting different than plain-text (ie. HTML, Markdown) when posting, relates to ``:instance, allowed_post_formats`` +* `collapse_message_with_subjects`: When a message has a subject(aka Content Warning), collapse it by default +* `hide_post_stats`: Hide notices statistics(repeats, favorites, …) +* `hide_user_stats`: Hide profile statistics(posts, posts per day, followers, followings, …) + +## :assets + +This section configures assets to be used with various frontends. Currently the only option +relates to mascots on the mastodon frontend + +* `mascots`: KeywordList of mascots, each element __MUST__ contain both a `url` and a + `mime_type` key. +* `default_mascot`: An element from `mascots` - This will be used as the default mascot + on MastoFE (default: `:pleroma_fox_tan`) + +## :mrf_simple +* `media_removal`: List of instances to remove medias from +* `media_nsfw`: List of instances to put medias as NSFW(sensitive) from +* `federated_timeline_removal`: List of instances to remove from Federated (aka The Whole Known Network) Timeline +* `reject`: List of instances to reject any activities from +* `accept`: List of instances to accept any activities from +* `report_removal`: List of instances to reject reports from +* `avatar_removal`: List of instances to strip avatars from +* `banner_removal`: List of instances to strip banners from + +## :mrf_subchain +This policy processes messages through an alternate pipeline when a given message matches certain criteria. +All criteria are configured as a map of regular expressions to lists of policy modules. + +* `match_actor`: Matches a series of regular expressions against the actor field. + +Example: + +``` +config :pleroma, :mrf_subchain, + match_actor: %{ + ~r/https:\/\/example.com/s => [Pleroma.Web.ActivityPub.MRF.DropPolicy] + } +``` + +## :mrf_rejectnonpublic +* `allow_followersonly`: whether to allow followers-only posts +* `allow_direct`: whether to allow direct messages + +## :mrf_hellthread +* `delist_threshold`: Number of mentioned users after which the message gets delisted (the message can still be seen, but it will not show up in public timelines and mentioned users won't get notifications about it). Set to 0 to disable. +* `reject_threshold`: Number of mentioned users after which the messaged gets rejected. Set to 0 to disable. + +## :mrf_keyword +* `reject`: A list of patterns which result in message being rejected, each pattern can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) +* `federated_timeline_removal`: A list of patterns which result in message being removed from federated timelines (a.k.a unlisted), each pattern can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) +* `replace`: A list of tuples containing `{pattern, replacement}`, `pattern` can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html) + +## :mrf_mention +* `actors`: A list of actors, for which to drop any posts mentioning. + +## :mrf_vocabulary +* `accept`: A list of ActivityStreams terms to accept. If empty, all supported messages are accepted. +* `reject`: A list of ActivityStreams terms to reject. If empty, no messages are rejected. + +## :media_proxy +* `enabled`: Enables proxying of remote media to the instance’s proxy +* `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts. +* `proxy_opts`: All options defined in `Pleroma.ReverseProxy` documentation, defaults to `[max_body_length: (25*1_048_576)]`. +* `whitelist`: List of domains to bypass the mediaproxy + +## :gopher +* `enabled`: Enables the gopher interface +* `ip`: IP address to bind to +* `port`: Port to bind to +* `dstport`: Port advertised in urls (optional, defaults to `port`) + +## Pleroma.Web.Endpoint +`Phoenix` endpoint configuration, all configuration options can be viewed [here](https://hexdocs.pm/phoenix/Phoenix.Endpoint.html#module-dynamic-configuration), only common options are listed here +* `http` - a list containing http protocol configuration, all configuration options can be viewed [here](https://hexdocs.pm/plug_cowboy/Plug.Cowboy.html#module-options), only common options are listed here. For deployment using docker, you need to set this to `[ip: {0,0,0,0}, port: 4000]` to make pleroma accessible from other containers (such as your nginx server). + - `ip` - a tuple consisting of 4 integers + - `port` +* `url` - a list containing the configuration for generating urls, accepts + - `host` - the host without the scheme and a post (e.g `example.com`, not `https://example.com:2020`) + - `scheme` - e.g `http`, `https` + - `port` + - `path` +* `extra_cookie_attrs` - a list of `Key=Value` strings to be added as non-standard cookie attributes. Defaults to `["SameSite=Lax"]`. See the [SameSite article](https://www.owasp.org/index.php/SameSite) on OWASP for more info. + + + +**Important note**: if you modify anything inside these lists, default `config.exs` values will be overwritten, which may result in breakage, to make sure this does not happen please copy the default value for the list from `config.exs` and modify/add only what you need + +Example: +```elixir +config :pleroma, Pleroma.Web.Endpoint, + url: [host: "example.com", port: 2020, scheme: "https"], + http: [ + # start copied from config.exs + dispatch: [ + {:_, + [ + {"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []}, + {"/websocket", Phoenix.Endpoint.CowboyWebSocket, + {Phoenix.Transports.WebSocket, + {Pleroma.Web.Endpoint, Pleroma.Web.UserSocket, websocket_config}}}, + {:_, Phoenix.Endpoint.Cowboy2Handler, {Pleroma.Web.Endpoint, []}} + ]} + # end copied from config.exs + ], + port: 8080, + ip: {127, 0, 0, 1} + ] +``` + +This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls starting with `https://example.com:2020` + +## :activitypub +* ``unfollow_blocked``: Whether blocks result in people getting unfollowed +* ``outgoing_blocks``: Whether to federate blocks to other instances +* ``deny_follow_blocked``: Whether to disallow following an account that has blocked the user in question +* ``sign_object_fetches``: Sign object fetches with HTTP signatures + +## :http_security +* ``enabled``: Whether the managed content security policy is enabled +* ``sts``: Whether to additionally send a `Strict-Transport-Security` header +* ``sts_max_age``: The maximum age for the `Strict-Transport-Security` header if sent +* ``ct_max_age``: The maximum age for the `Expect-CT` header if sent +* ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"` +* ``report_uri``: Adds the specified url to `report-uri` and `report-to` group in CSP header. + +## :mrf_user_allowlist + +The keys in this section are the domain names that the policy should apply to. +Each key should be assigned a list of users that should be allowed through by +their ActivityPub ID. + +An example: + +```elixir +config :pleroma, :mrf_user_allowlist, + "example.org": ["https://example.org/users/admin"] +``` + +## :web_push_encryption, :vapid_details + +Web Push Notifications configuration. You can use the mix task `mix web_push.gen.keypair` to generate it. + +* ``subject``: a mailto link for the administrative contact. It’s best if this email is not a personal email address, but rather a group email so that if a person leaves an organization, is unavailable for an extended period, or otherwise can’t respond, someone else on the list can. +* ``public_key``: VAPID public key +* ``private_key``: VAPID private key + +## Pleroma.Captcha +* `enabled`: Whether the captcha should be shown on registration +* `method`: The method/service to use for captcha +* `seconds_valid`: The time in seconds for which the captcha is valid + +### Pleroma.Captcha.Kocaptcha +Kocaptcha is a very simple captcha service with a single API endpoint, +the source code is here: https://github.com/koto-bank/kocaptcha. The default endpoint +`https://captcha.kotobank.ch` is hosted by the developer. + +* `endpoint`: the kocaptcha endpoint to use + +## :admin_token + +Allows to set a token that can be used to authenticate with the admin api without using an actual user by giving it as the 'admin_token' parameter. Example: + +```elixir +config :pleroma, :admin_token, "somerandomtoken" +``` + +You can then do + +```sh +curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerandomtoken" +``` + +## Oban + +[Oban](https://github.com/sorentwo/oban) asynchronous job processor configuration. + +Configuration options described in [Oban readme](https://github.com/sorentwo/oban#usage): +* `repo` - app's Ecto repo (`Pleroma.Repo`) +* `verbose` - logs verbosity +* `prune` - non-retryable jobs [pruning settings](https://github.com/sorentwo/oban#pruning) (`:disabled` / `{:maxlen, value}` / `{:maxage, value}`) +* `queues` - job queues (see below) + +Pleroma has the following queues: + +* `activity_expiration` - Activity expiration +* `federator_outgoing` - Outgoing federation +* `federator_incoming` - Incoming federation +* `mailer` - Email sender, see [`Pleroma.Emails.Mailer`](#pleromaemailsmailer) +* `transmogrifier` - Transmogrifier +* `web_push` - Web push notifications +* `scheduled_activities` - Scheduled activities, see [`Pleroma.ScheduledActivity`](#pleromascheduledactivity) + +Example: + +```elixir +config :pleroma, Oban, + repo: Pleroma.Repo, + verbose: false, + prune: {:maxlen, 1500}, + queues: [ + federator_incoming: 50, + federator_outgoing: 50 + ] +``` + +This config contains two queues: `federator_incoming` and `federator_outgoing`. Both have the number of max concurrent jobs set to `50`. + +### Migrating `pleroma_job_queue` settings + +`config :pleroma_job_queue, :queues` is replaced by `config :pleroma, Oban, :queues` and uses the same format (keys are queues' names, values are max concurrent jobs numbers). + +### Note on running with PostgreSQL in silent mode + +If you are running PostgreSQL in [`silent_mode`](https://postgresqlco.nf/en/doc/param/silent_mode?version=9.1), it's advised to set [`log_destination`](https://postgresqlco.nf/en/doc/param/log_destination?version=9.1) to `syslog`, +otherwise `postmaster.log` file may grow because of "you don't own a lock of type ShareLock" warnings (see https://github.com/sorentwo/oban/issues/52). + +## :workers + +Includes custom worker options not interpretable directly by `Oban`. + +* `retries` — keyword lists where keys are `Oban` queues (see above) and values are numbers of max attempts for failed jobs. + +Example: + +```elixir +config :pleroma, :workers, + retries: [ + federator_incoming: 5, + federator_outgoing: 5 + ] +``` + +### Migrating `Pleroma.Web.Federator.RetryQueue` settings + +* `max_retries` is replaced with `config :pleroma, :workers, retries: [federator_outgoing: 5]` +* `enabled: false` corresponds to `config :pleroma, :workers, retries: [federator_outgoing: 1]` +* deprecated options: `max_jobs`, `initial_timeout` + +## Pleroma.Web.Metadata +* `providers`: a list of metadata providers to enable. Providers available: + * Pleroma.Web.Metadata.Providers.OpenGraph + * Pleroma.Web.Metadata.Providers.TwitterCard + * Pleroma.Web.Metadata.Providers.RelMe - add links from user bio with rel=me into the `
` as `` +* `unfurl_nsfw`: If set to `true` nsfw attachments will be shown in previews + +## :rich_media +* `enabled`: if enabled the instance will parse metadata from attached links to generate link previews +* `ignore_hosts`: list of hosts which will be ignored by the metadata parser. For example `["accounts.google.com", "xss.website"]`, defaults to `[]`. +* `ignore_tld`: list TLDs (top-level domains) which will ignore for parse metadata. default is ["local", "localdomain", "lan"] +* `parsers`: list of Rich Media parsers + +## :fetch_initial_posts +* `enabled`: if enabled, when a new user is federated with, fetch some of their latest posts +* `pages`: the amount of pages to fetch + +## :hackney_pools + +Advanced. Tweaks Hackney (http client) connections pools. + +There's three pools used: + +* `:federation` for the federation jobs. + You may want this pool max_connections to be at least equal to the number of federator jobs + retry queue jobs. +* `:media` for rich media, media proxy +* `:upload` for uploaded media (if using a remote uploader and `proxy_remote: true`) + +For each pool, the options are: + +* `max_connections` - how much connections a pool can hold +* `timeout` - retention duration for connections + +## :auto_linker + +Configuration for the `auto_linker` library: + +* `class: "auto-linker"` - specify the class to be added to the generated link. false to clear +* `rel: "noopener noreferrer"` - override the rel attribute. false to clear +* `new_window: true` - set to false to remove `target='_blank'` attribute +* `scheme: false` - Set to true to link urls with schema `http://google.com` +* `truncate: false` - Set to a number to truncate urls longer then the number. Truncated urls will end in `..` +* `strip_prefix: true` - Strip the scheme prefix +* `extra: false` - link urls with rarely used schemes (magnet, ipfs, irc, etc.) + +Example: + +```elixir +config :auto_linker, + opts: [ + scheme: true, + extra: true, + class: false, + strip_prefix: false, + new_window: false, + rel: false + ] +``` + +## Pleroma.Scheduler + +Configuration for [Quantum](https://github.com/quantum-elixir/quantum-core) jobs scheduler. + +See [Quantum readme](https://github.com/quantum-elixir/quantum-core#usage) for the list of supported options. + +Example: + +```elixir +config :pleroma, Pleroma.Scheduler, + global: true, + overlap: true, + timezone: :utc, + jobs: [{"0 */6 * * * *", {Pleroma.Web.Websub, :refresh_subscriptions, []}}] +``` + +The above example defines a single job which invokes `Pleroma.Web.Websub.refresh_subscriptions()` every 6 hours ("0 */6 * * * *", [crontab format](https://en.wikipedia.org/wiki/Cron)). + +## Pleroma.ScheduledActivity + +* `daily_user_limit`: the number of scheduled activities a user is allowed to create in a single day (Default: `25`) +* `total_user_limit`: the number of scheduled activities a user is allowed to create in total (Default: `300`) +* `enabled`: whether scheduled activities are sent to the job queue to be executed + +## Pleroma.ActivityExpiration + +# `enabled`: whether expired activities will be sent to the job queue to be deleted + +## Pleroma.Web.Auth.Authenticator + +* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator +* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication + +## :ldap + +Use LDAP for user authentication. When a user logs in to the Pleroma +instance, the name and password will be verified by trying to authenticate +(bind) to an LDAP server. If a user exists in the LDAP directory but there +is no account with the same name yet on the Pleroma instance then a new +Pleroma account will be created with the same name as the LDAP user name. + +* `enabled`: enables LDAP authentication +* `host`: LDAP server hostname +* `port`: LDAP port, e.g. 389 or 636 +* `ssl`: true to use SSL, usually implies the port 636 +* `sslopts`: additional SSL options +* `tls`: true to start TLS, usually implies the port 389 +* `tlsopts`: additional TLS options +* `base`: LDAP base, e.g. "dc=example,dc=com" +* `uid`: LDAP attribute name to authenticate the user, e.g. when "cn", the filter will be "cn=username,base" + +## BBS / SSH access + +To enable simple command line interface accessible over ssh, add a setting like this to your configuration file: + +```exs +app_dir = File.cwd! +priv_dir = Path.join([app_dir, "priv/ssh_keys"]) + +config :esshd, + enabled: true, + priv_dir: priv_dir, + handler: "Pleroma.BBS.Handler", + port: 10_022, + password_authenticator: "Pleroma.BBS.Authenticator" +``` + +Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -m PEM -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT` + +## :auth + +* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator +* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication + +Authentication / authorization settings. + +* `auth_template`: authentication form template. By default it's `show.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/show.html.eex`. +* `oauth_consumer_template`: OAuth consumer mode authentication form template. By default it's `consumer.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex`. +* `oauth_consumer_strategies`: the list of enabled OAuth consumer strategies; by default it's set by `OAUTH_CONSUMER_STRATEGIES` environment variable. Each entry in this space-delimited string should be of format `` or `:` (e.g. `twitter` or `keycloak:ueberauth_keycloak_strategy` in case dependency is named differently than `ueberauth_`). + +## :email_notifications + +Email notifications settings. + + - digest - emails of "what you've missed" for users who have been + inactive for a while. + - active: globally enable or disable digest emails + - schedule: When to send digest email, in [crontab format](https://en.wikipedia.org/wiki/Cron). + "0 0 * * 0" is the default, meaning "once a week at midnight on Sunday morning" + - interval: Minimum interval between digest emails to one user + - inactivity_threshold: Minimum user inactivity threshold + +## Pleroma.Emails.UserEmail + +- `:logo` - a path to a custom logo. Set it to `nil` to use the default Pleroma logo. +- `:styling` - a map with color settings for email templates. + +## OAuth consumer mode + +OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.). +Implementation is based on Ueberauth; see the list of [available strategies](https://github.com/ueberauth/ueberauth/wiki/List-of-Strategies). + +Note: each strategy is shipped as a separate dependency; in order to get the strategies, run `OAUTH_CONSUMER_STRATEGIES="..." mix deps.get`, +e.g. `OAUTH_CONSUMER_STRATEGIES="twitter facebook google microsoft" mix deps.get`. +The server should also be started with `OAUTH_CONSUMER_STRATEGIES="..." mix phx.server` in case you enable any strategies. + +Note: each strategy requires separate setup (on external provider side and Pleroma side). Below are the guidelines on setting up most popular strategies. + +Note: make sure that `"SameSite=Lax"` is set in `extra_cookie_attrs` when you have this feature enabled. OAuth consumer mode will not work with `"SameSite=Strict"` + +* For Twitter, [register an app](https://developer.twitter.com/en/apps), configure callback URL to https:///oauth/twitter/callback + +* For Facebook, [register an app](https://developers.facebook.com/apps), configure callback URL to https:///oauth/facebook/callback, enable Facebook Login service at https://developers.facebook.com/apps//fb-login/settings/ + +* For Google, [register an app](https://console.developers.google.com), configure callback URL to https:///oauth/google/callback + +* For Microsoft, [register an app](https://portal.azure.com), configure callback URL to https:///oauth/microsoft/callback + +Once the app is configured on external OAuth provider side, add app's credentials and strategy-specific settings (if any — e.g. see Microsoft below) to `config/prod.secret.exs`, +per strategy's documentation (e.g. [ueberauth_twitter](https://github.com/ueberauth/ueberauth_twitter)). Example config basing on environment variables: + +```elixir +# Twitter +config :ueberauth, Ueberauth.Strategy.Twitter.OAuth, + consumer_key: System.get_env("TWITTER_CONSUMER_KEY"), + consumer_secret: System.get_env("TWITTER_CONSUMER_SECRET") + +# Facebook +config :ueberauth, Ueberauth.Strategy.Facebook.OAuth, + client_id: System.get_env("FACEBOOK_APP_ID"), + client_secret: System.get_env("FACEBOOK_APP_SECRET"), + redirect_uri: System.get_env("FACEBOOK_REDIRECT_URI") + +# Google +config :ueberauth, Ueberauth.Strategy.Google.OAuth, + client_id: System.get_env("GOOGLE_CLIENT_ID"), + client_secret: System.get_env("GOOGLE_CLIENT_SECRET"), + redirect_uri: System.get_env("GOOGLE_REDIRECT_URI") + +# Microsoft +config :ueberauth, Ueberauth.Strategy.Microsoft.OAuth, + client_id: System.get_env("MICROSOFT_CLIENT_ID"), + client_secret: System.get_env("MICROSOFT_CLIENT_SECRET") + +config :ueberauth, Ueberauth, + providers: [ + microsoft: {Ueberauth.Strategy.Microsoft, [callback_params: []]} + ] + +# Keycloak +# Note: make sure to add `keycloak:ueberauth_keycloak_strategy` entry to `OAUTH_CONSUMER_STRATEGIES` environment variable +keycloak_url = "https://publicly-reachable-keycloak-instance.org:8080" + +config :ueberauth, Ueberauth.Strategy.Keycloak.OAuth, + client_id: System.get_env("KEYCLOAK_CLIENT_ID"), + client_secret: System.get_env("KEYCLOAK_CLIENT_SECRET"), + site: keycloak_url, + authorize_url: "#{keycloak_url}/auth/realms/master/protocol/openid-connect/auth", + token_url: "#{keycloak_url}/auth/realms/master/protocol/openid-connect/token", + userinfo_url: "#{keycloak_url}/auth/realms/master/protocol/openid-connect/userinfo", + token_method: :post + +config :ueberauth, Ueberauth, + providers: [ + keycloak: {Ueberauth.Strategy.Keycloak, [uid_field: :email]} + ] +``` + +## OAuth 2.0 provider - :oauth2 + +Configure OAuth 2 provider capabilities: + +* `token_expires_in` - The lifetime in seconds of the access token. +* `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token. +* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`. +* `clean_expired_tokens_interval` - Interval to run the job to clean expired tokens. Defaults to `86_400_000` (24 hours). + +## :emoji +* `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]` +* `pack_extensions`: A list of file extensions for emojis, when no emoji.txt for a pack is present. Example `[".png", ".gif"]` +* `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]` +* `default_manifest`: Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download. Currently only one manifest can be added (no arrays). + +## Database options + +### RUM indexing for full text search +* `rum_enabled`: If RUM indexes should be used. Defaults to `false`. + +RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. While they may eventually be mainlined, for now they have to be installed as a PostgreSQL extension from https://github.com/postgrespro/rum. + +Their advantage over the standard GIN indexes is that they allow efficient ordering of search results by timestamp, which makes search queries a lot faster on larger servers, by one or two orders of magnitude. They take up around 3 times as much space as GIN indexes. + +To enable them, both the `rum_enabled` flag has to be set and the following special migration has to be run: + +`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/` + +This will probably take a long time. + +## :rate_limit + +This is an advanced feature and disabled by default. + +A keyword list of rate limiters where a key is a limiter name and value is the limiter configuration. The basic configuration is a tuple where: + +* The first element: `scale` (Integer). The time scale in milliseconds. +* The second element: `limit` (Integer). How many requests to limit in the time scale provided. + +It is also possible to have different limits for unauthenticated and authenticated users: the keyword value must be a list of two tuples where the first one is a config for unauthenticated users and the second one is for authenticated. + +See [`Pleroma.Plugs.RateLimiter`](Pleroma.Plugs.RateLimiter.html) documentation for examples. + +Supported rate limiters: + +* `:search` for the search requests (account & status search etc.) +* `:app_account_creation` for registering user accounts from the same IP address +* `:relations_actions` for actions on relations with all users (follow, unfollow) +* `:relation_id_action` for actions on relation with a specific user (follow, unfollow) +* `:statuses_actions` for create / delete / fav / unfav / reblog / unreblog actions on any statuses +* `:status_id_action` for fav / unfav or reblog / unreblog actions on the same status by the same user + +## :web_cache_ttl + +The expiration time for the web responses cache. Values should be in milliseconds or `nil` to disable expiration. + +Available caches: + +* `:activity_pub` - activity pub routes (except question activities). Defaults to `nil` (no expiration). +* `:activity_pub_question` - activity pub routes (question activities). Defaults to `30_000` (30 seconds). From b4cf74c1067b866574a63fbd25ccb12cc1fed619 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Sun, 15 Sep 2019 14:53:58 +0300 Subject: [PATCH 34/45] added prepare html for RichMedia.Parser --- lib/pleroma/web/rich_media/parser.ex | 6 +++++- mix.exs | 2 +- mix.lock | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index f5f9e358c..c06b0a0f2 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -81,6 +81,7 @@ defmodule Pleroma.Web.RichMedia.Parser do {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options) html + |> parse_html |> maybe_parse() |> Map.put(:url, url) |> clean_parsed_data() @@ -91,6 +92,8 @@ defmodule Pleroma.Web.RichMedia.Parser do end end + defp parse_html(html), do: Floki.parse(html) + defp maybe_parse(html) do Enum.reduce_while(parsers(), %{}, fn parser, acc -> case parser.parse(html, acc) do @@ -100,7 +103,8 @@ defmodule Pleroma.Web.RichMedia.Parser do end) end - defp check_parsed_data(%{title: title} = data) when is_binary(title) and byte_size(title) > 0 do + defp check_parsed_data(%{title: title} = data) + when is_binary(title) and byte_size(title) > 0 do {:ok, data} end diff --git a/mix.exs b/mix.exs index dfa530358..6f952fa12 100644 --- a/mix.exs +++ b/mix.exs @@ -131,7 +131,7 @@ defmodule Pleroma.Mixfile do {:phoenix_swoosh, "~> 0.2"}, {:gen_smtp, "~> 0.13"}, {:websocket_client, git: "https://github.com/jeremyong/websocket_client.git", only: :test}, - {:floki, "~> 0.20.0"}, + {:floki, "~> 0.23.0"}, {:ex_syslogger, github: "slashmili/ex_syslogger", tag: "1.4.0"}, {:timex, "~> 3.5"}, {:ueberauth, "~> 0.4"}, diff --git a/mix.lock b/mix.lock index 5762dae4f..2bce00dea 100644 --- a/mix.lock +++ b/mix.lock @@ -34,7 +34,7 @@ "ex_rated": {:hex, :ex_rated, "1.3.3", "30ecbdabe91f7eaa9d37fa4e81c85ba420f371babeb9d1910adbcd79ec798d27", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm"}, "ex_syslogger": {:git, "https://github.com/slashmili/ex_syslogger.git", "f3963399047af17e038897c69e20d552e6899e1d", [tag: "1.4.0"]}, "excoveralls": {:hex, :excoveralls, "0.11.1", "dd677fbdd49114fdbdbf445540ec735808250d56b011077798316505064edb2c", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, - "floki": {:hex, :floki, "0.20.4", "be42ac911fece24b4c72f3b5846774b6e61b83fe685c2fc9d62093277fb3bc86", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}, {:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, + "floki": {:hex, :floki, "0.23.0", "956ab6dba828c96e732454809fb0bd8d43ce0979b75f34de6322e73d4c917829", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm"}, "gen_smtp": {:hex, :gen_smtp, "0.14.0", "39846a03522456077c6429b4badfd1d55e5e7d0fdfb65e935b7c5e38549d9202", [:rebar3], [], "hexpm"}, "gettext": {:hex, :gettext, "0.17.0", "abe21542c831887a2b16f4c94556db9c421ab301aee417b7c4fbde7fbdbe01ec", [:mix], [], "hexpm"}, "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, From ca88e37a8f3b0b52771f94df676e26471fb44019 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 16 Sep 2019 12:55:05 +0700 Subject: [PATCH 35/45] Fix a race condition in tests --- test/web/mastodon_api/mastodon_api_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 9c5322ccb..fb04748bb 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -752,7 +752,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do query_string = "ids[]=#{id1}&ids[]=#{id2}" conn = get(conn, "/api/v1/statuses/?#{query_string}") - assert [%{"id" => ^id1}, %{"id" => ^id2}] = json_response(conn, :ok) + assert [%{"id" => ^id1}, %{"id" => ^id2}] = Enum.sort_by(json_response(conn, :ok), & &1["id"]) end describe "deleting a status" do From aab264db82054df470075c65ca25c42bbcc5d7a8 Mon Sep 17 00:00:00 2001 From: Steven Fuchs Date: Mon, 16 Sep 2019 07:44:03 +0000 Subject: [PATCH 36/45] Streamer refactoring --- .gitignore | 4 + config/config.exs | 4 + lib/pleroma/activity/ir/topics.ex | 63 ++++ lib/pleroma/application.ex | 2 +- lib/pleroma/notification.ex | 6 +- lib/pleroma/web/activity_pub/activity_pub.ex | 48 +-- .../web/mastodon_api/websocket_handler.ex | 7 +- lib/pleroma/web/streamer.ex | 318 ------------------ lib/pleroma/web/streamer/ping.ex | 33 ++ lib/pleroma/web/streamer/state.ex | 68 ++++ lib/pleroma/web/streamer/streamer.ex | 55 +++ lib/pleroma/web/streamer/streamer_socket.ex | 31 ++ lib/pleroma/web/streamer/supervisor.ex | 33 ++ lib/pleroma/web/streamer/worker.ex | 220 ++++++++++++ lib/pleroma/web/views/streamer_view.ex | 66 ++++ mix.exs | 1 + mix.lock | 1 + test/activity/ir/topics_test.exs | 141 ++++++++ test/integration/mastodon_websocket_test.exs | 16 +- test/notification_test.exs | 11 +- test/support/conn_case.ex | 4 + test/support/data_case.ex | 4 + test/web/activity_pub/activity_pub_test.exs | 4 +- test/web/streamer/ping_test.exs | 36 ++ test/web/streamer/state_test.exs | 54 +++ test/web/{ => streamer}/streamer_test.exs | 105 +++--- 26 files changed, 888 insertions(+), 447 deletions(-) create mode 100644 lib/pleroma/activity/ir/topics.ex delete mode 100644 lib/pleroma/web/streamer.ex create mode 100644 lib/pleroma/web/streamer/ping.ex create mode 100644 lib/pleroma/web/streamer/state.ex create mode 100644 lib/pleroma/web/streamer/streamer.ex create mode 100644 lib/pleroma/web/streamer/streamer_socket.ex create mode 100644 lib/pleroma/web/streamer/supervisor.ex create mode 100644 lib/pleroma/web/streamer/worker.ex create mode 100644 lib/pleroma/web/views/streamer_view.ex create mode 100644 test/activity/ir/topics_test.exs create mode 100644 test/web/streamer/ping_test.exs create mode 100644 test/web/streamer/state_test.exs rename test/web/{ => streamer}/streamer_test.exs (86%) diff --git a/.gitignore b/.gitignore index 4e71a7df0..3b0c7d361 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,7 @@ docs/generated_config.md # Code test coverage /cover /Elixir.*.coverdata + +.idea +pleroma.iml + diff --git a/config/config.exs b/config/config.exs index ab6e00c98..b1b98af93 100644 --- a/config/config.exs +++ b/config/config.exs @@ -331,6 +331,10 @@ config :pleroma, :activitypub, follow_handshake_timeout: 500, sign_object_fetches: true +config :pleroma, :streamer, + workers: 3, + overflow_workers: 2 + config :pleroma, :user, deny_follow_blocked: true config :pleroma, :mrf_normalize_markup, scrub_policy: Pleroma.HTML.Scrubber.Default diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex new file mode 100644 index 000000000..010897abc --- /dev/null +++ b/lib/pleroma/activity/ir/topics.ex @@ -0,0 +1,63 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Activity.Ir.Topics do + alias Pleroma.Object + alias Pleroma.Web.ActivityPub.Visibility + + def get_activity_topics(activity) do + activity + |> Object.normalize() + |> generate_topics(activity) + |> List.flatten() + end + + defp generate_topics(%{data: %{"type" => "Answer"}}, _) do + [] + end + + defp generate_topics(object, activity) do + ["user", "list"] ++ visibility_tags(object, activity) + end + + defp visibility_tags(object, activity) do + case Visibility.get_visibility(activity) do + "public" -> + if activity.local do + ["public", "public:local"] + else + ["public"] + end + |> item_creation_tags(object, activity) + + "direct" -> + ["direct"] + + _ -> + [] + end + end + + defp item_creation_tags(tags, %{data: %{"type" => "Create"}} = object, activity) do + tags ++ hashtags_to_topics(object) ++ attachment_topics(object, activity) + end + + defp item_creation_tags(tags, _, _) do + tags + end + + defp hashtags_to_topics(%{data: %{"tag" => tags}}) do + tags + |> Enum.filter(&is_bitstring(&1)) + |> Enum.map(fn tag -> "hashtag:" <> tag end) + end + + defp hashtags_to_topics(_), do: [] + + defp attachment_topics(%{data: %{"attachment" => []}}, _act), do: [] + + defp attachment_topics(_object, %{local: true}), do: ["public:media", "public:local:media"] + + defp attachment_topics(_object, _act), do: ["public:media"] +end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 49094704b..3b37ce630 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -141,7 +141,7 @@ defmodule Pleroma.Application do defp streamer_child(:test), do: [] defp streamer_child(_) do - [Pleroma.Web.Streamer] + [Pleroma.Web.Streamer.supervisor()] end defp oauth_cleanup_child(true), diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index b7c880c51..8012389ac 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -210,8 +210,10 @@ defmodule Pleroma.Notification do unless skip?(activity, user) do notification = %Notification{user_id: user.id, activity: activity} {:ok, notification} = Repo.insert(notification) - Streamer.stream("user", notification) - Streamer.stream("user:notification", notification) + + ["user", "user:notification"] + |> Streamer.stream(notification) + Push.send(notification) notification end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 41f6a0f1f..bc5ae7fbf 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Activity + alias Pleroma.Activity.Ir.Topics alias Pleroma.Config alias Pleroma.Conversation alias Pleroma.Notification @@ -16,6 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.User alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.Streamer alias Pleroma.Web.WebFinger alias Pleroma.Workers.BackgroundWorker @@ -187,9 +189,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do participations |> Repo.preload(:user) - Enum.each(participations, fn participation -> - Pleroma.Web.Streamer.stream("participation", participation) - end) + Streamer.stream("participation", participations) end def stream_out_participations(%Object{data: %{"context" => context}}, user) do @@ -208,41 +208,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def stream_out_participations(_, _), do: :noop - def stream_out(activity) do - if activity.data["type"] in ["Create", "Announce", "Delete"] do - object = Object.normalize(activity) - # Do not stream out poll replies - unless object.data["type"] == "Answer" do - Pleroma.Web.Streamer.stream("user", activity) - Pleroma.Web.Streamer.stream("list", activity) + def stream_out(%Activity{data: %{"type" => data_type}} = activity) + when data_type in ["Create", "Announce", "Delete"] do + activity + |> Topics.get_activity_topics() + |> Streamer.stream(activity) + end - if get_visibility(activity) == "public" do - Pleroma.Web.Streamer.stream("public", activity) - - if activity.local do - Pleroma.Web.Streamer.stream("public:local", activity) - end - - if activity.data["type"] in ["Create"] do - object.data - |> Map.get("tag", []) - |> Enum.filter(fn tag -> is_bitstring(tag) end) - |> Enum.each(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end) - - if object.data["attachment"] != [] do - Pleroma.Web.Streamer.stream("public:media", activity) - - if activity.local do - Pleroma.Web.Streamer.stream("public:local:media", activity) - end - end - end - else - if get_visibility(activity) == "direct", - do: Pleroma.Web.Streamer.stream("direct", activity) - end - end - end + def stream_out(_activity) do + :noop end def create(%{to: to, actor: actor, context: context, object: object} = params, fake \\ false) do diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index dbd3542ea..3c26eb406 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.OAuth.Token + alias Pleroma.Web.Streamer @behaviour :cowboy_websocket @@ -24,7 +25,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do ] @anonymous_streams ["public", "public:local", "hashtag"] - # Handled by periodic keepalive in Pleroma.Web.Streamer. + # Handled by periodic keepalive in Pleroma.Web.Streamer.Ping. @timeout :infinity def init(%{qs: qs} = req, state) do @@ -65,7 +66,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do }, topic #{state.topic}" ) - Pleroma.Web.Streamer.add_socket(state.topic, streamer_socket(state)) + Streamer.add_socket(state.topic, streamer_socket(state)) {:ok, state} end @@ -80,7 +81,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do }, topic #{state.topic || "?"}: #{inspect(reason)}" ) - Pleroma.Web.Streamer.remove_socket(state.topic, streamer_socket(state)) + Streamer.remove_socket(state.topic, streamer_socket(state)) :ok end diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex deleted file mode 100644 index 587c43f40..000000000 --- a/lib/pleroma/web/streamer.ex +++ /dev/null @@ -1,318 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Streamer do - use GenServer - require Logger - alias Pleroma.Activity - alias Pleroma.Config - alias Pleroma.Conversation.Participation - alias Pleroma.Notification - alias Pleroma.Object - alias Pleroma.User - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.ActivityPub.Visibility - alias Pleroma.Web.CommonAPI - alias Pleroma.Web.MastodonAPI.NotificationView - - @keepalive_interval :timer.seconds(30) - - def start_link(_) do - GenServer.start_link(__MODULE__, %{}, name: __MODULE__) - end - - def add_socket(topic, socket) do - GenServer.cast(__MODULE__, %{action: :add, socket: socket, topic: topic}) - end - - def remove_socket(topic, socket) do - GenServer.cast(__MODULE__, %{action: :remove, socket: socket, topic: topic}) - end - - def stream(topic, item) do - GenServer.cast(__MODULE__, %{action: :stream, topic: topic, item: item}) - end - - def init(args) do - Process.send_after(self(), %{action: :ping}, @keepalive_interval) - - {:ok, args} - end - - def handle_info(%{action: :ping}, topics) do - topics - |> Map.values() - |> List.flatten() - |> Enum.each(fn socket -> - Logger.debug("Sending keepalive ping") - send(socket.transport_pid, {:text, ""}) - end) - - Process.send_after(self(), %{action: :ping}, @keepalive_interval) - - {:noreply, topics} - end - - def handle_cast(%{action: :stream, topic: "direct", item: item}, topics) do - recipient_topics = - User.get_recipients_from_activity(item) - |> Enum.map(fn %{id: id} -> "direct:#{id}" end) - - Enum.each(recipient_topics || [], fn user_topic -> - Logger.debug("Trying to push direct message to #{user_topic}\n\n") - push_to_socket(topics, user_topic, item) - end) - - {:noreply, topics} - end - - def handle_cast(%{action: :stream, topic: "participation", item: participation}, topics) do - user_topic = "direct:#{participation.user_id}" - Logger.debug("Trying to push a conversation participation to #{user_topic}\n\n") - - push_to_socket(topics, user_topic, participation) - - {:noreply, topics} - end - - def handle_cast(%{action: :stream, topic: "list", item: item}, topics) do - # filter the recipient list if the activity is not public, see #270. - recipient_lists = - case Visibility.is_public?(item) do - true -> - Pleroma.List.get_lists_from_activity(item) - - _ -> - Pleroma.List.get_lists_from_activity(item) - |> Enum.filter(fn list -> - owner = User.get_cached_by_id(list.user_id) - - Visibility.visible_for_user?(item, owner) - end) - end - - recipient_topics = - recipient_lists - |> Enum.map(fn %{id: id} -> "list:#{id}" end) - - Enum.each(recipient_topics || [], fn list_topic -> - Logger.debug("Trying to push message to #{list_topic}\n\n") - push_to_socket(topics, list_topic, item) - end) - - {:noreply, topics} - end - - def handle_cast( - %{action: :stream, topic: topic, item: %Notification{} = item}, - topics - ) - when topic in ["user", "user:notification"] do - topics - |> Map.get("#{topic}:#{item.user_id}", []) - |> Enum.each(fn socket -> - with %User{} = user <- User.get_cached_by_ap_id(socket.assigns[:user].ap_id), - true <- should_send?(user, item) do - send( - socket.transport_pid, - {:text, represent_notification(socket.assigns[:user], item)} - ) - end - end) - - {:noreply, topics} - end - - def handle_cast(%{action: :stream, topic: "user", item: item}, topics) do - Logger.debug("Trying to push to users") - - recipient_topics = - User.get_recipients_from_activity(item) - |> Enum.map(fn %{id: id} -> "user:#{id}" end) - - Enum.each(recipient_topics, fn topic -> - push_to_socket(topics, topic, item) - end) - - {:noreply, topics} - end - - def handle_cast(%{action: :stream, topic: topic, item: item}, topics) do - Logger.debug("Trying to push to #{topic}") - Logger.debug("Pushing item to #{topic}") - push_to_socket(topics, topic, item) - {:noreply, topics} - end - - def handle_cast(%{action: :add, topic: topic, socket: socket}, sockets) do - topic = internal_topic(topic, socket) - sockets_for_topic = sockets[topic] || [] - sockets_for_topic = Enum.uniq([socket | sockets_for_topic]) - sockets = Map.put(sockets, topic, sockets_for_topic) - Logger.debug("Got new conn for #{topic}") - {:noreply, sockets} - end - - def handle_cast(%{action: :remove, topic: topic, socket: socket}, sockets) do - topic = internal_topic(topic, socket) - sockets_for_topic = sockets[topic] || [] - sockets_for_topic = List.delete(sockets_for_topic, socket) - sockets = Map.put(sockets, topic, sockets_for_topic) - Logger.debug("Removed conn for #{topic}") - {:noreply, sockets} - end - - def handle_cast(m, state) do - Logger.info("Unknown: #{inspect(m)}, #{inspect(state)}") - {:noreply, state} - end - - defp represent_update(%Activity{} = activity, %User{} = user) do - %{ - event: "update", - payload: - Pleroma.Web.MastodonAPI.StatusView.render( - "status.json", - activity: activity, - for: user - ) - |> Jason.encode!() - } - |> Jason.encode!() - end - - defp represent_update(%Activity{} = activity) do - %{ - event: "update", - payload: - Pleroma.Web.MastodonAPI.StatusView.render( - "status.json", - activity: activity - ) - |> Jason.encode!() - } - |> Jason.encode!() - end - - def represent_conversation(%Participation{} = participation) do - %{ - event: "conversation", - payload: - Pleroma.Web.MastodonAPI.ConversationView.render("participation.json", %{ - participation: participation, - for: participation.user - }) - |> Jason.encode!() - } - |> Jason.encode!() - end - - @spec represent_notification(User.t(), Notification.t()) :: binary() - defp represent_notification(%User{} = user, %Notification{} = notify) do - %{ - event: "notification", - payload: - NotificationView.render( - "show.json", - %{notification: notify, for: user} - ) - |> Jason.encode!() - } - |> Jason.encode!() - end - - defp should_send?(%User{} = user, %Activity{} = item) do - blocks = user.info.blocks || [] - mutes = user.info.mutes || [] - reblog_mutes = user.info.muted_reblogs || [] - domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(user.info.domain_blocks) - - with parent when not is_nil(parent) <- Object.normalize(item), - true <- Enum.all?([blocks, mutes, reblog_mutes], &(item.actor not in &1)), - true <- Enum.all?([blocks, mutes], &(parent.data["actor"] not in &1)), - %{host: item_host} <- URI.parse(item.actor), - %{host: parent_host} <- URI.parse(parent.data["actor"]), - false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host), - false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, parent_host), - true <- thread_containment(item, user), - false <- CommonAPI.thread_muted?(user, item) do - true - else - _ -> false - end - end - - defp should_send?(%User{} = user, %Notification{activity: activity}) do - should_send?(user, activity) - end - - def push_to_socket(topics, topic, %Activity{data: %{"type" => "Announce"}} = item) do - Enum.each(topics[topic] || [], fn socket -> - # Get the current user so we have up-to-date blocks etc. - if socket.assigns[:user] do - user = User.get_cached_by_ap_id(socket.assigns[:user].ap_id) - - if should_send?(user, item) do - send(socket.transport_pid, {:text, represent_update(item, user)}) - end - else - send(socket.transport_pid, {:text, represent_update(item)}) - end - end) - end - - def push_to_socket(topics, topic, %Participation{} = participation) do - Enum.each(topics[topic] || [], fn socket -> - send(socket.transport_pid, {:text, represent_conversation(participation)}) - end) - end - - def push_to_socket(topics, topic, %Activity{ - data: %{"type" => "Delete", "deleted_activity_id" => deleted_activity_id} - }) do - Enum.each(topics[topic] || [], fn socket -> - send( - socket.transport_pid, - {:text, %{event: "delete", payload: to_string(deleted_activity_id)} |> Jason.encode!()} - ) - end) - end - - def push_to_socket(_topics, _topic, %Activity{data: %{"type" => "Delete"}}), do: :noop - - def push_to_socket(topics, topic, item) do - Enum.each(topics[topic] || [], fn socket -> - # Get the current user so we have up-to-date blocks etc. - if socket.assigns[:user] do - user = User.get_cached_by_ap_id(socket.assigns[:user].ap_id) - blocks = user.info.blocks || [] - mutes = user.info.mutes || [] - - with true <- Enum.all?([blocks, mutes], &(item.actor not in &1)), - true <- thread_containment(item, user) do - send(socket.transport_pid, {:text, represent_update(item, user)}) - end - else - send(socket.transport_pid, {:text, represent_update(item)}) - end - end) - end - - defp internal_topic(topic, socket) when topic in ~w[user user:notification direct] do - "#{topic}:#{socket.assigns[:user].id}" - end - - defp internal_topic(topic, _), do: topic - - @spec thread_containment(Activity.t(), User.t()) :: boolean() - defp thread_containment(_activity, %User{info: %{skip_thread_containment: true}}), do: true - - defp thread_containment(activity, user) do - if Config.get([:instance, :skip_thread_containment]) do - true - else - ActivityPub.contain_activity(activity, user) - end - end -end diff --git a/lib/pleroma/web/streamer/ping.ex b/lib/pleroma/web/streamer/ping.ex new file mode 100644 index 000000000..f77cbb95c --- /dev/null +++ b/lib/pleroma/web/streamer/ping.ex @@ -0,0 +1,33 @@ +defmodule Pleroma.Web.Streamer.Ping do + use GenServer + require Logger + + alias Pleroma.Web.Streamer.State + alias Pleroma.Web.Streamer.StreamerSocket + + @keepalive_interval :timer.seconds(30) + + def start_link(opts) do + ping_interval = Keyword.get(opts, :ping_interval, @keepalive_interval) + GenServer.start_link(__MODULE__, %{ping_interval: ping_interval}, name: __MODULE__) + end + + def init(%{ping_interval: ping_interval} = args) do + Process.send_after(self(), :ping, ping_interval) + {:ok, args} + end + + def handle_info(:ping, %{ping_interval: ping_interval} = state) do + State.get_sockets() + |> Map.values() + |> List.flatten() + |> Enum.each(fn %StreamerSocket{transport_pid: transport_pid} -> + Logger.debug("Sending keepalive ping") + send(transport_pid, {:text, ""}) + end) + + Process.send_after(self(), :ping, ping_interval) + + {:noreply, state} + end +end diff --git a/lib/pleroma/web/streamer/state.ex b/lib/pleroma/web/streamer/state.ex new file mode 100644 index 000000000..7b5199068 --- /dev/null +++ b/lib/pleroma/web/streamer/state.ex @@ -0,0 +1,68 @@ +defmodule Pleroma.Web.Streamer.State do + use GenServer + require Logger + + alias Pleroma.Web.Streamer.StreamerSocket + + def start_link(_) do + GenServer.start_link(__MODULE__, %{sockets: %{}}, name: __MODULE__) + end + + def add_socket(topic, socket) do + GenServer.call(__MODULE__, {:add, socket, topic}) + end + + def remove_socket(topic, socket) do + GenServer.call(__MODULE__, {:remove, socket, topic}) + end + + def get_sockets do + %{sockets: stream_sockets} = GenServer.call(__MODULE__, :get_state) + stream_sockets + end + + def init(init_arg) do + {:ok, init_arg} + end + + def handle_call(:get_state, _from, state) do + {:reply, state, state} + end + + def handle_call({:add, socket, topic}, _from, %{sockets: sockets} = state) do + internal_topic = internal_topic(topic, socket) + stream_socket = StreamerSocket.from_socket(socket) + + sockets_for_topic = + sockets + |> Map.get(internal_topic, []) + |> List.insert_at(0, stream_socket) + |> Enum.uniq() + + state = put_in(state, [:sockets, internal_topic], sockets_for_topic) + Logger.debug("Got new conn for #{topic}") + {:reply, state, state} + end + + def handle_call({:remove, socket, topic}, _from, %{sockets: sockets} = state) do + internal_topic = internal_topic(topic, socket) + stream_socket = StreamerSocket.from_socket(socket) + + sockets_for_topic = + sockets + |> Map.get(internal_topic, []) + |> List.delete(stream_socket) + + state = Kernel.put_in(state, [:sockets, internal_topic], sockets_for_topic) + {:reply, state, state} + end + + defp internal_topic(topic, socket) + when topic in ~w[user user:notification direct] do + "#{topic}:#{socket.assigns[:user].id}" + end + + defp internal_topic(topic, _) do + topic + end +end diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex new file mode 100644 index 000000000..8cf719277 --- /dev/null +++ b/lib/pleroma/web/streamer/streamer.ex @@ -0,0 +1,55 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Streamer do + alias Pleroma.Web.Streamer.State + alias Pleroma.Web.Streamer.Worker + + @timeout 60_000 + @mix_env Mix.env() + + def add_socket(topic, socket) do + State.add_socket(topic, socket) + end + + def remove_socket(topic, socket) do + State.remove_socket(topic, socket) + end + + def get_sockets do + State.get_sockets() + end + + def stream(topics, items) do + if should_send?() do + Task.async(fn -> + :poolboy.transaction( + :streamer_worker, + &Worker.stream(&1, topics, items), + @timeout + ) + end) + end + end + + def supervisor, do: Pleroma.Web.Streamer.Supervisor + + defp should_send? do + handle_should_send(@mix_env) + end + + defp handle_should_send(:test) do + case Process.whereis(:streamer_worker) do + nil -> + false + + pid -> + Process.alive?(pid) + end + end + + defp handle_should_send(_) do + true + end +end diff --git a/lib/pleroma/web/streamer/streamer_socket.ex b/lib/pleroma/web/streamer/streamer_socket.ex new file mode 100644 index 000000000..f006c0306 --- /dev/null +++ b/lib/pleroma/web/streamer/streamer_socket.ex @@ -0,0 +1,31 @@ +defmodule Pleroma.Web.Streamer.StreamerSocket do + defstruct transport_pid: nil, user: nil + + alias Pleroma.User + alias Pleroma.Web.Streamer.StreamerSocket + + def from_socket(%{ + transport_pid: transport_pid, + assigns: %{user: nil} + }) do + %StreamerSocket{ + transport_pid: transport_pid + } + end + + def from_socket(%{ + transport_pid: transport_pid, + assigns: %{user: %User{} = user} + }) do + %StreamerSocket{ + transport_pid: transport_pid, + user: user + } + end + + def from_socket(%{transport_pid: transport_pid}) do + %StreamerSocket{ + transport_pid: transport_pid + } + end +end diff --git a/lib/pleroma/web/streamer/supervisor.ex b/lib/pleroma/web/streamer/supervisor.ex new file mode 100644 index 000000000..6afe19323 --- /dev/null +++ b/lib/pleroma/web/streamer/supervisor.ex @@ -0,0 +1,33 @@ +defmodule Pleroma.Web.Streamer.Supervisor do + use Supervisor + + def start_link(opts) do + Supervisor.start_link(__MODULE__, opts, name: __MODULE__) + end + + def init(args) do + children = [ + {Pleroma.Web.Streamer.State, args}, + {Pleroma.Web.Streamer.Ping, args}, + :poolboy.child_spec(:streamer_worker, poolboy_config()) + ] + + opts = [strategy: :one_for_one, name: Pleroma.Web.Streamer.Supervisor] + Supervisor.init(children, opts) + end + + defp poolboy_config do + opts = + Pleroma.Config.get(:streamer, + workers: 3, + overflow_workers: 2 + ) + + [ + {:name, {:local, :streamer_worker}}, + {:worker_module, Pleroma.Web.Streamer.Worker}, + {:size, opts[:workers]}, + {:max_overflow, opts[:overflow_workers]} + ] + end +end diff --git a/lib/pleroma/web/streamer/worker.ex b/lib/pleroma/web/streamer/worker.ex new file mode 100644 index 000000000..5804508eb --- /dev/null +++ b/lib/pleroma/web/streamer/worker.ex @@ -0,0 +1,220 @@ +defmodule Pleroma.Web.Streamer.Worker do + use GenServer + + require Logger + + alias Pleroma.Activity + alias Pleroma.Config + alias Pleroma.Conversation.Participation + alias Pleroma.Notification + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.Streamer.State + alias Pleroma.Web.Streamer.StreamerSocket + alias Pleroma.Web.StreamerView + + def start_link(_) do + GenServer.start_link(__MODULE__, %{}, []) + end + + def init(init_arg) do + {:ok, init_arg} + end + + def stream(pid, topics, items) do + GenServer.call(pid, {:stream, topics, items}) + end + + def handle_call({:stream, topics, item}, _from, state) when is_list(topics) do + Enum.each(topics, fn t -> + do_stream(%{topic: t, item: item}) + end) + + {:reply, state, state} + end + + def handle_call({:stream, topic, items}, _from, state) when is_list(items) do + Enum.each(items, fn i -> + do_stream(%{topic: topic, item: i}) + end) + + {:reply, state, state} + end + + def handle_call({:stream, topic, item}, _from, state) do + do_stream(%{topic: topic, item: item}) + + {:reply, state, state} + end + + defp do_stream(%{topic: "direct", item: item}) do + recipient_topics = + User.get_recipients_from_activity(item) + |> Enum.map(fn %{id: id} -> "direct:#{id}" end) + + Enum.each(recipient_topics, fn user_topic -> + Logger.debug("Trying to push direct message to #{user_topic}\n\n") + push_to_socket(State.get_sockets(), user_topic, item) + end) + end + + defp do_stream(%{topic: "participation", item: participation}) do + user_topic = "direct:#{participation.user_id}" + Logger.debug("Trying to push a conversation participation to #{user_topic}\n\n") + + push_to_socket(State.get_sockets(), user_topic, participation) + end + + defp do_stream(%{topic: "list", item: item}) do + # filter the recipient list if the activity is not public, see #270. + recipient_lists = + case Visibility.is_public?(item) do + true -> + Pleroma.List.get_lists_from_activity(item) + + _ -> + Pleroma.List.get_lists_from_activity(item) + |> Enum.filter(fn list -> + owner = User.get_cached_by_id(list.user_id) + + Visibility.visible_for_user?(item, owner) + end) + end + + recipient_topics = + recipient_lists + |> Enum.map(fn %{id: id} -> "list:#{id}" end) + + Enum.each(recipient_topics, fn list_topic -> + Logger.debug("Trying to push message to #{list_topic}\n\n") + push_to_socket(State.get_sockets(), list_topic, item) + end) + end + + defp do_stream(%{topic: topic, item: %Notification{} = item}) + when topic in ["user", "user:notification"] do + State.get_sockets() + |> Map.get("#{topic}:#{item.user_id}", []) + |> Enum.each(fn %StreamerSocket{transport_pid: transport_pid, user: socket_user} -> + with %User{} = user <- User.get_cached_by_ap_id(socket_user.ap_id), + true <- should_send?(user, item) do + send(transport_pid, {:text, StreamerView.render("notification.json", socket_user, item)}) + end + end) + end + + defp do_stream(%{topic: "user", item: item}) do + Logger.debug("Trying to push to users") + + recipient_topics = + User.get_recipients_from_activity(item) + |> Enum.map(fn %{id: id} -> "user:#{id}" end) + + Enum.each(recipient_topics, fn topic -> + push_to_socket(State.get_sockets(), topic, item) + end) + end + + defp do_stream(%{topic: topic, item: item}) do + Logger.debug("Trying to push to #{topic}") + Logger.debug("Pushing item to #{topic}") + push_to_socket(State.get_sockets(), topic, item) + end + + defp should_send?(%User{} = user, %Activity{} = item) do + blocks = user.info.blocks || [] + mutes = user.info.mutes || [] + reblog_mutes = user.info.muted_reblogs || [] + domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(user.info.domain_blocks) + + with parent when not is_nil(parent) <- Object.normalize(item), + true <- Enum.all?([blocks, mutes, reblog_mutes], &(item.actor not in &1)), + true <- Enum.all?([blocks, mutes], &(parent.data["actor"] not in &1)), + %{host: item_host} <- URI.parse(item.actor), + %{host: parent_host} <- URI.parse(parent.data["actor"]), + false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host), + false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, parent_host), + true <- thread_containment(item, user), + false <- CommonAPI.thread_muted?(user, item) do + true + else + _ -> false + end + end + + defp should_send?(%User{} = user, %Notification{activity: activity}) do + should_send?(user, activity) + end + + def push_to_socket(topics, topic, %Activity{data: %{"type" => "Announce"}} = item) do + Enum.each(topics[topic] || [], fn %StreamerSocket{ + transport_pid: transport_pid, + user: socket_user + } -> + # Get the current user so we have up-to-date blocks etc. + if socket_user do + user = User.get_cached_by_ap_id(socket_user.ap_id) + + if should_send?(user, item) do + send(transport_pid, {:text, StreamerView.render("update.json", item, user)}) + end + else + send(transport_pid, {:text, StreamerView.render("update.json", item)}) + end + end) + end + + def push_to_socket(topics, topic, %Participation{} = participation) do + Enum.each(topics[topic] || [], fn %StreamerSocket{transport_pid: transport_pid} -> + send(transport_pid, {:text, StreamerView.render("conversation.json", participation)}) + end) + end + + def push_to_socket(topics, topic, %Activity{ + data: %{"type" => "Delete", "deleted_activity_id" => deleted_activity_id} + }) do + Enum.each(topics[topic] || [], fn %StreamerSocket{transport_pid: transport_pid} -> + send( + transport_pid, + {:text, %{event: "delete", payload: to_string(deleted_activity_id)} |> Jason.encode!()} + ) + end) + end + + def push_to_socket(_topics, _topic, %Activity{data: %{"type" => "Delete"}}), do: :noop + + def push_to_socket(topics, topic, item) do + Enum.each(topics[topic] || [], fn %StreamerSocket{ + transport_pid: transport_pid, + user: socket_user + } -> + # Get the current user so we have up-to-date blocks etc. + if socket_user do + user = User.get_cached_by_ap_id(socket_user.ap_id) + blocks = user.info.blocks || [] + mutes = user.info.mutes || [] + + with true <- Enum.all?([blocks, mutes], &(item.actor not in &1)), + true <- thread_containment(item, user) do + send(transport_pid, {:text, StreamerView.render("update.json", item, user)}) + end + else + send(transport_pid, {:text, StreamerView.render("update.json", item)}) + end + end) + end + + @spec thread_containment(Activity.t(), User.t()) :: boolean() + defp thread_containment(_activity, %User{info: %{skip_thread_containment: true}}), do: true + + defp thread_containment(activity, user) do + if Config.get([:instance, :skip_thread_containment]) do + true + else + ActivityPub.contain_activity(activity, user) + end + end +end diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex new file mode 100644 index 000000000..b13030fa0 --- /dev/null +++ b/lib/pleroma/web/views/streamer_view.ex @@ -0,0 +1,66 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.StreamerView do + use Pleroma.Web, :view + + alias Pleroma.Activity + alias Pleroma.Conversation.Participation + alias Pleroma.Notification + alias Pleroma.User + alias Pleroma.Web.MastodonAPI.NotificationView + + def render("update.json", %Activity{} = activity, %User{} = user) do + %{ + event: "update", + payload: + Pleroma.Web.MastodonAPI.StatusView.render( + "status.json", + activity: activity, + for: user + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + + def render("notification.json", %User{} = user, %Notification{} = notify) do + %{ + event: "notification", + payload: + NotificationView.render( + "show.json", + %{notification: notify, for: user} + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + + def render("update.json", %Activity{} = activity) do + %{ + event: "update", + payload: + Pleroma.Web.MastodonAPI.StatusView.render( + "status.json", + activity: activity + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + + def render("conversation.json", %Participation{} = participation) do + %{ + event: "conversation", + payload: + Pleroma.Web.MastodonAPI.ConversationView.render("participation.json", %{ + participation: participation, + for: participation.user + }) + |> Jason.encode!() + } + |> Jason.encode!() + end +end diff --git a/mix.exs b/mix.exs index f1e98585b..911ebad1d 100644 --- a/mix.exs +++ b/mix.exs @@ -144,6 +144,7 @@ defmodule Pleroma.Mixfile do git: "https://git.pleroma.social/pleroma/http_signatures.git", ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"}, {:telemetry, "~> 0.3"}, + {:poolboy, "~> 1.5"}, {:prometheus_ex, "~> 3.0"}, {:prometheus_plugs, "~> 1.1"}, {:prometheus_phoenix, "~> 1.3"}, diff --git a/mix.lock b/mix.lock index 41697dd5c..0bf6a811e 100644 --- a/mix.lock +++ b/mix.lock @@ -73,6 +73,7 @@ "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"}, "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, + "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "prometheus": {:hex, :prometheus, "4.4.1", "1e96073b3ed7788053768fea779cbc896ddc3bdd9ba60687f2ad50b252ac87d6", [:mix, :rebar3], [], "hexpm"}, "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/activity/ir/topics_test.exs b/test/activity/ir/topics_test.exs new file mode 100644 index 000000000..e75f83586 --- /dev/null +++ b/test/activity/ir/topics_test.exs @@ -0,0 +1,141 @@ +defmodule Pleroma.Activity.Ir.TopicsTest do + use Pleroma.DataCase + + alias Pleroma.Activity + alias Pleroma.Activity.Ir.Topics + alias Pleroma.Object + + require Pleroma.Constants + + describe "poll answer" do + test "produce no topics" do + activity = %Activity{object: %Object{data: %{"type" => "Answer"}}} + + assert [] == Topics.get_activity_topics(activity) + end + end + + describe "non poll answer" do + test "always add user and list topics" do + activity = %Activity{object: %Object{data: %{"type" => "FooBar"}}} + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "user") + assert Enum.member?(topics, "list") + end + end + + describe "public visibility" do + setup do + activity = %Activity{ + object: %Object{data: %{"type" => "Note"}}, + data: %{"to" => [Pleroma.Constants.as_public()]} + } + + {:ok, activity: activity} + end + + test "produces public topic", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public") + end + + test "local action produces public:local topic", %{activity: activity} do + activity = %{activity | local: true} + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public:local") + end + + test "non-local action does not produce public:local topic", %{activity: activity} do + activity = %{activity | local: false} + topics = Topics.get_activity_topics(activity) + + refute Enum.member?(topics, "public:local") + end + end + + describe "public visibility create events" do + setup do + activity = %Activity{ + object: %Object{data: %{"type" => "Create", "attachment" => []}}, + data: %{"to" => [Pleroma.Constants.as_public()]} + } + + {:ok, activity: activity} + end + + test "with no attachments doesn't produce public:media topics", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + refute Enum.member?(topics, "public:media") + refute Enum.member?(topics, "public:local:media") + end + + test "converts tags to hash tags", %{activity: %{object: %{data: data} = object} = activity} do + tagged_data = Map.put(data, "tag", ["foo", "bar"]) + activity = %{activity | object: %{object | data: tagged_data}} + + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "hashtag:foo") + assert Enum.member?(topics, "hashtag:bar") + end + + test "only converts strinngs to hash tags", %{ + activity: %{object: %{data: data} = object} = activity + } do + tagged_data = Map.put(data, "tag", [2]) + activity = %{activity | object: %{object | data: tagged_data}} + + topics = Topics.get_activity_topics(activity) + + refute Enum.member?(topics, "hashtag:2") + end + end + + describe "public visibility create events with attachments" do + setup do + activity = %Activity{ + object: %Object{data: %{"type" => "Create", "attachment" => ["foo"]}}, + data: %{"to" => [Pleroma.Constants.as_public()]} + } + + {:ok, activity: activity} + end + + test "produce public:media topics", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public:media") + end + + test "local produces public:local:media topics", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public:local:media") + end + + test "non-local doesn't produce public:local:media topics", %{activity: activity} do + activity = %{activity | local: false} + + topics = Topics.get_activity_topics(activity) + + refute Enum.member?(topics, "public:local:media") + end + end + + describe "non-public visibility" do + test "produces direct topic" do + activity = %Activity{object: %Object{data: %{"type" => "Note"}}, data: %{"to" => []}} + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "direct") + refute Enum.member?(topics, "public") + refute Enum.member?(topics, "public:local") + refute Enum.member?(topics, "public:media") + refute Enum.member?(topics, "public:local:media") + end + end +end diff --git a/test/integration/mastodon_websocket_test.exs b/test/integration/mastodon_websocket_test.exs index 63bf73412..c04262808 100644 --- a/test/integration/mastodon_websocket_test.exs +++ b/test/integration/mastodon_websocket_test.exs @@ -11,7 +11,6 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do alias Pleroma.Integration.WebsocketClient alias Pleroma.Web.CommonAPI alias Pleroma.Web.OAuth - alias Pleroma.Web.Streamer @path Pleroma.Web.Endpoint.url() |> URI.parse() @@ -19,16 +18,6 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do |> Map.put(:path, "/api/v1/streaming") |> URI.to_string() - setup do - GenServer.start(Streamer, %{}, name: Streamer) - - on_exit(fn -> - if pid = Process.whereis(Streamer) do - Process.exit(pid, :kill) - end - end) - end - def start_socket(qs \\ nil, headers \\ []) do path = case qs do @@ -53,12 +42,14 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) end + @tag needs_streamer: true test "allows public streams without authentication" do assert {:ok, _} = start_socket("?stream=public") assert {:ok, _} = start_socket("?stream=public:local") assert {:ok, _} = start_socket("?stream=hashtag&tag=lain") end + @tag needs_streamer: true test "receives well formatted events" do user = insert(:user) {:ok, _} = start_socket("?stream=public") @@ -103,6 +94,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert {:ok, _} = start_socket("?stream=user&access_token=#{state.token.token}") end + @tag needs_streamer: true test "accepts the 'user' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user&access_token=#{token.token}") @@ -111,6 +103,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) =~ ":badarg" end + @tag needs_streamer: true test "accepts the 'user:notification' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user:notification&access_token=#{token.token}") @@ -119,6 +112,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) =~ ":badarg" end + @tag needs_streamer: true test "accepts valid token on Sec-WebSocket-Protocol header", %{token: token} do assert {:ok, _} = start_socket("?stream=user", [{"Sec-WebSocket-Protocol", token.token}]) diff --git a/test/notification_test.exs b/test/notification_test.exs index 3be9db09b..3d2f9a8fc 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -69,16 +69,7 @@ defmodule Pleroma.NotificationTest do end describe "create_notification" do - setup do - GenServer.start(Streamer, %{}, name: Streamer) - - on_exit(fn -> - if pid = Process.whereis(Streamer) do - Process.exit(pid, :kill) - end - end) - end - + @tag needs_streamer: true test "it creates a notification for user and send to the 'user' and the 'user:notification' stream" do user = insert(:user) task = Task.async(fn -> assert_receive {:text, _}, 4_000 end) diff --git a/test/support/conn_case.ex b/test/support/conn_case.ex index ec5892ff5..b39c70677 100644 --- a/test/support/conn_case.ex +++ b/test/support/conn_case.ex @@ -40,6 +40,10 @@ defmodule Pleroma.Web.ConnCase do Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, {:shared, self()}) end + if tags[:needs_streamer] do + start_supervised(Pleroma.Web.Streamer.supervisor()) + end + {:ok, conn: Phoenix.ConnTest.build_conn()} end end diff --git a/test/support/data_case.ex b/test/support/data_case.ex index f3d98e7e3..17fa15214 100644 --- a/test/support/data_case.ex +++ b/test/support/data_case.ex @@ -39,6 +39,10 @@ defmodule Pleroma.DataCase do Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, {:shared, self()}) end + if tags[:needs_streamer] do + start_supervised(Pleroma.Web.Streamer.supervisor()) + end + :ok end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index d0118fefa..4100108a5 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -38,9 +38,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do stream: fn _, _ -> nil end do ActivityPub.stream_out_participations(conversation.participations) - Enum.each(participations, fn participation -> - assert called(Pleroma.Web.Streamer.stream("participation", participation)) - end) + assert called(Pleroma.Web.Streamer.stream("participation", participations)) end end end diff --git a/test/web/streamer/ping_test.exs b/test/web/streamer/ping_test.exs new file mode 100644 index 000000000..3d52c00e4 --- /dev/null +++ b/test/web/streamer/ping_test.exs @@ -0,0 +1,36 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PingTest do + use Pleroma.DataCase + + import Pleroma.Factory + alias Pleroma.Web.Streamer + + setup do + start_supervised({Streamer.supervisor(), [ping_interval: 30]}) + + :ok + end + + describe "sockets" do + setup do + user = insert(:user) + {:ok, %{user: user}} + end + + test "it sends pings", %{user: user} do + task = + Task.async(fn -> + assert_receive {:text, received_event}, 40 + assert_receive {:text, received_event}, 40 + assert_receive {:text, received_event}, 40 + end) + + Streamer.add_socket("public", %{transport_pid: task.pid, assigns: %{user: user}}) + + Task.await(task) + end + end +end diff --git a/test/web/streamer/state_test.exs b/test/web/streamer/state_test.exs new file mode 100644 index 000000000..d1aeac541 --- /dev/null +++ b/test/web/streamer/state_test.exs @@ -0,0 +1,54 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.StateTest do + use Pleroma.DataCase + + import Pleroma.Factory + alias Pleroma.Web.Streamer + alias Pleroma.Web.Streamer.StreamerSocket + + @moduletag needs_streamer: true + + describe "sockets" do + setup do + user = insert(:user) + user2 = insert(:user) + {:ok, %{user: user, user2: user2}} + end + + test "it can add a socket", %{user: user} do + Streamer.add_socket("public", %{transport_pid: 1, assigns: %{user: user}}) + + assert(%{"public" => [%StreamerSocket{transport_pid: 1}]} = Streamer.get_sockets()) + end + + test "it can add multiple sockets per user", %{user: user} do + Streamer.add_socket("public", %{transport_pid: 1, assigns: %{user: user}}) + Streamer.add_socket("public", %{transport_pid: 2, assigns: %{user: user}}) + + assert( + %{ + "public" => [ + %StreamerSocket{transport_pid: 2}, + %StreamerSocket{transport_pid: 1} + ] + } = Streamer.get_sockets() + ) + end + + test "it will not add a duplicate socket", %{user: user} do + Streamer.add_socket("activity", %{transport_pid: 1, assigns: %{user: user}}) + Streamer.add_socket("activity", %{transport_pid: 1, assigns: %{user: user}}) + + assert( + %{ + "activity" => [ + %StreamerSocket{transport_pid: 1} + ] + } = Streamer.get_sockets() + ) + end + end +end diff --git a/test/web/streamer_test.exs b/test/web/streamer/streamer_test.exs similarity index 86% rename from test/web/streamer_test.exs rename to test/web/streamer/streamer_test.exs index 96fa7645f..88847e20f 100644 --- a/test/web/streamer_test.exs +++ b/test/web/streamer/streamer_test.exs @@ -5,24 +5,20 @@ defmodule Pleroma.Web.StreamerTest do use Pleroma.DataCase + import Pleroma.Factory + alias Pleroma.List alias Pleroma.User alias Pleroma.Web.CommonAPI alias Pleroma.Web.Streamer - import Pleroma.Factory + alias Pleroma.Web.Streamer.StreamerSocket + alias Pleroma.Web.Streamer.Worker + @moduletag needs_streamer: true clear_config_all([:instance, :skip_thread_containment]) describe "user streams" do setup do - GenServer.start(Streamer, %{}, name: Streamer) - - on_exit(fn -> - if pid = Process.whereis(Streamer) do - Process.exit(pid, :kill) - end - end) - user = insert(:user) notify = insert(:notification, user: user, activity: build(:note_activity)) {:ok, %{user: user, notify: notify}} @@ -125,11 +121,9 @@ defmodule Pleroma.Web.StreamerTest do assert_receive {:text, _}, 4_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user - } + user: user } {:ok, activity} = CommonAPI.post(other_user, %{"status" => "Test"}) @@ -138,7 +132,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) @@ -155,11 +149,9 @@ defmodule Pleroma.Web.StreamerTest do assert received_event == expected_event end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user - } + user: user } {:ok, activity} = CommonAPI.delete(activity.id, other_user) @@ -168,7 +160,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) end @@ -189,9 +181,9 @@ defmodule Pleroma.Web.StreamerTest do ) task = Task.async(fn -> refute_receive {:text, _}, 1_000 end) - fake_socket = %{transport_pid: task.pid, assigns: %{user: user}} + fake_socket = %StreamerSocket{transport_pid: task.pid, user: user} topics = %{"public" => [fake_socket]} - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) end @@ -211,9 +203,9 @@ defmodule Pleroma.Web.StreamerTest do ) task = Task.async(fn -> assert_receive {:text, _}, 1_000 end) - fake_socket = %{transport_pid: task.pid, assigns: %{user: user}} + fake_socket = %StreamerSocket{transport_pid: task.pid, user: user} topics = %{"public" => [fake_socket]} - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) end @@ -233,9 +225,9 @@ defmodule Pleroma.Web.StreamerTest do ) task = Task.async(fn -> assert_receive {:text, _}, 1_000 end) - fake_socket = %{transport_pid: task.pid, assigns: %{user: user}} + fake_socket = %StreamerSocket{transport_pid: task.pid, user: user} topics = %{"public" => [fake_socket]} - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) end @@ -251,11 +243,9 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user - } + user: user } {:ok, activity} = CommonAPI.post(blocked_user, %{"status" => "Test"}) @@ -264,7 +254,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) end @@ -284,11 +274,9 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user_a - } + user: user_a } {:ok, activity} = @@ -301,7 +289,7 @@ defmodule Pleroma.Web.StreamerTest do "list:#{list.id}" => [fake_socket] } - Streamer.handle_cast(%{action: :stream, topic: "list", item: activity}, topics) + Worker.handle_call({:stream, "list", activity}, self(), topics) Task.await(task) end @@ -318,11 +306,9 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user_a - } + user: user_a } {:ok, activity} = @@ -335,12 +321,12 @@ defmodule Pleroma.Web.StreamerTest do "list:#{list.id}" => [fake_socket] } - Streamer.handle_cast(%{action: :stream, topic: "list", item: activity}, topics) + Worker.handle_call({:stream, "list", activity}, self(), topics) Task.await(task) end - test "it send wanted private posts to list" do + test "it sends wanted private posts to list" do user_a = insert(:user) user_b = insert(:user) @@ -354,11 +340,9 @@ defmodule Pleroma.Web.StreamerTest do assert_receive {:text, _}, 1_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user_a - } + user: user_a } {:ok, activity} = @@ -367,11 +351,12 @@ defmodule Pleroma.Web.StreamerTest do "visibility" => "private" }) - topics = %{ - "list:#{list.id}" => [fake_socket] - } + Streamer.add_socket( + "list:#{list.id}", + fake_socket + ) - Streamer.handle_cast(%{action: :stream, topic: "list", item: activity}, topics) + Worker.handle_call({:stream, "list", activity}, self(), %{}) Task.await(task) end @@ -387,11 +372,9 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user1 - } + user: user1 } {:ok, create_activity} = CommonAPI.post(user3, %{"status" => "I'm kawen"}) @@ -401,7 +384,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Streamer.push_to_socket(topics, "public", announce_activity) + Worker.push_to_socket(topics, "public", announce_activity) Task.await(task) end @@ -417,6 +400,8 @@ defmodule Pleroma.Web.StreamerTest do task = Task.async(fn -> refute_receive {:text, _}, 4_000 end) + Process.sleep(4000) + Streamer.add_socket( "user", %{transport_pid: task.pid, assigns: %{user: user2}} @@ -428,14 +413,6 @@ defmodule Pleroma.Web.StreamerTest do describe "direct streams" do setup do - GenServer.start(Streamer, %{}, name: Streamer) - - on_exit(fn -> - if pid = Process.whereis(Streamer) do - Process.exit(pid, :kill) - end - end) - :ok end @@ -480,6 +457,8 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 4_000 end) + Process.sleep(1000) + Streamer.add_socket( "direct", %{transport_pid: task.pid, assigns: %{user: user}} @@ -521,6 +500,8 @@ defmodule Pleroma.Web.StreamerTest do assert last_status["id"] == to_string(create_activity.id) end) + Process.sleep(1000) + Streamer.add_socket( "direct", %{transport_pid: task.pid, assigns: %{user: user}} From c623b4324deaf236334a0f77a81435b5bffadf3c Mon Sep 17 00:00:00 2001 From: kaniini Date: Mon, 16 Sep 2019 09:09:21 +0000 Subject: [PATCH 37/45] Revert "Merge branch 'streamer-refactoring' into 'develop'" This reverts merge request !1653 --- .gitignore | 4 - config/config.exs | 4 - lib/pleroma/activity/ir/topics.ex | 63 ---- lib/pleroma/application.ex | 2 +- lib/pleroma/notification.ex | 6 +- lib/pleroma/web/activity_pub/activity_pub.ex | 48 ++- .../web/mastodon_api/websocket_handler.ex | 7 +- lib/pleroma/web/streamer.ex | 318 ++++++++++++++++++ lib/pleroma/web/streamer/ping.ex | 33 -- lib/pleroma/web/streamer/state.ex | 68 ---- lib/pleroma/web/streamer/streamer.ex | 55 --- lib/pleroma/web/streamer/streamer_socket.ex | 31 -- lib/pleroma/web/streamer/supervisor.ex | 33 -- lib/pleroma/web/streamer/worker.ex | 220 ------------ lib/pleroma/web/views/streamer_view.ex | 66 ---- mix.exs | 1 - mix.lock | 1 - test/activity/ir/topics_test.exs | 141 -------- test/integration/mastodon_websocket_test.exs | 16 +- test/notification_test.exs | 11 +- test/support/conn_case.ex | 4 - test/support/data_case.ex | 4 - test/web/activity_pub/activity_pub_test.exs | 4 +- test/web/streamer/ping_test.exs | 36 -- test/web/streamer/state_test.exs | 54 --- test/web/{streamer => }/streamer_test.exs | 105 +++--- 26 files changed, 447 insertions(+), 888 deletions(-) delete mode 100644 lib/pleroma/activity/ir/topics.ex create mode 100644 lib/pleroma/web/streamer.ex delete mode 100644 lib/pleroma/web/streamer/ping.ex delete mode 100644 lib/pleroma/web/streamer/state.ex delete mode 100644 lib/pleroma/web/streamer/streamer.ex delete mode 100644 lib/pleroma/web/streamer/streamer_socket.ex delete mode 100644 lib/pleroma/web/streamer/supervisor.ex delete mode 100644 lib/pleroma/web/streamer/worker.ex delete mode 100644 lib/pleroma/web/views/streamer_view.ex delete mode 100644 test/activity/ir/topics_test.exs delete mode 100644 test/web/streamer/ping_test.exs delete mode 100644 test/web/streamer/state_test.exs rename test/web/{streamer => }/streamer_test.exs (86%) diff --git a/.gitignore b/.gitignore index 3b0c7d361..4e71a7df0 100644 --- a/.gitignore +++ b/.gitignore @@ -43,7 +43,3 @@ docs/generated_config.md # Code test coverage /cover /Elixir.*.coverdata - -.idea -pleroma.iml - diff --git a/config/config.exs b/config/config.exs index b1b98af93..ab6e00c98 100644 --- a/config/config.exs +++ b/config/config.exs @@ -331,10 +331,6 @@ config :pleroma, :activitypub, follow_handshake_timeout: 500, sign_object_fetches: true -config :pleroma, :streamer, - workers: 3, - overflow_workers: 2 - config :pleroma, :user, deny_follow_blocked: true config :pleroma, :mrf_normalize_markup, scrub_policy: Pleroma.HTML.Scrubber.Default diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex deleted file mode 100644 index 010897abc..000000000 --- a/lib/pleroma/activity/ir/topics.ex +++ /dev/null @@ -1,63 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Activity.Ir.Topics do - alias Pleroma.Object - alias Pleroma.Web.ActivityPub.Visibility - - def get_activity_topics(activity) do - activity - |> Object.normalize() - |> generate_topics(activity) - |> List.flatten() - end - - defp generate_topics(%{data: %{"type" => "Answer"}}, _) do - [] - end - - defp generate_topics(object, activity) do - ["user", "list"] ++ visibility_tags(object, activity) - end - - defp visibility_tags(object, activity) do - case Visibility.get_visibility(activity) do - "public" -> - if activity.local do - ["public", "public:local"] - else - ["public"] - end - |> item_creation_tags(object, activity) - - "direct" -> - ["direct"] - - _ -> - [] - end - end - - defp item_creation_tags(tags, %{data: %{"type" => "Create"}} = object, activity) do - tags ++ hashtags_to_topics(object) ++ attachment_topics(object, activity) - end - - defp item_creation_tags(tags, _, _) do - tags - end - - defp hashtags_to_topics(%{data: %{"tag" => tags}}) do - tags - |> Enum.filter(&is_bitstring(&1)) - |> Enum.map(fn tag -> "hashtag:" <> tag end) - end - - defp hashtags_to_topics(_), do: [] - - defp attachment_topics(%{data: %{"attachment" => []}}, _act), do: [] - - defp attachment_topics(_object, %{local: true}), do: ["public:media", "public:local:media"] - - defp attachment_topics(_object, _act), do: ["public:media"] -end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 3b37ce630..49094704b 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -141,7 +141,7 @@ defmodule Pleroma.Application do defp streamer_child(:test), do: [] defp streamer_child(_) do - [Pleroma.Web.Streamer.supervisor()] + [Pleroma.Web.Streamer] end defp oauth_cleanup_child(true), diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 8012389ac..b7c880c51 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -210,10 +210,8 @@ defmodule Pleroma.Notification do unless skip?(activity, user) do notification = %Notification{user_id: user.id, activity: activity} {:ok, notification} = Repo.insert(notification) - - ["user", "user:notification"] - |> Streamer.stream(notification) - + Streamer.stream("user", notification) + Streamer.stream("user:notification", notification) Push.send(notification) notification end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index bc5ae7fbf..41f6a0f1f 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -4,7 +4,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Activity - alias Pleroma.Activity.Ir.Topics alias Pleroma.Config alias Pleroma.Conversation alias Pleroma.Notification @@ -17,7 +16,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.User alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.Transmogrifier - alias Pleroma.Web.Streamer alias Pleroma.Web.WebFinger alias Pleroma.Workers.BackgroundWorker @@ -189,7 +187,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do participations |> Repo.preload(:user) - Streamer.stream("participation", participations) + Enum.each(participations, fn participation -> + Pleroma.Web.Streamer.stream("participation", participation) + end) end def stream_out_participations(%Object{data: %{"context" => context}}, user) do @@ -208,15 +208,41 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def stream_out_participations(_, _), do: :noop - def stream_out(%Activity{data: %{"type" => data_type}} = activity) - when data_type in ["Create", "Announce", "Delete"] do - activity - |> Topics.get_activity_topics() - |> Streamer.stream(activity) - end + def stream_out(activity) do + if activity.data["type"] in ["Create", "Announce", "Delete"] do + object = Object.normalize(activity) + # Do not stream out poll replies + unless object.data["type"] == "Answer" do + Pleroma.Web.Streamer.stream("user", activity) + Pleroma.Web.Streamer.stream("list", activity) - def stream_out(_activity) do - :noop + if get_visibility(activity) == "public" do + Pleroma.Web.Streamer.stream("public", activity) + + if activity.local do + Pleroma.Web.Streamer.stream("public:local", activity) + end + + if activity.data["type"] in ["Create"] do + object.data + |> Map.get("tag", []) + |> Enum.filter(fn tag -> is_bitstring(tag) end) + |> Enum.each(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end) + + if object.data["attachment"] != [] do + Pleroma.Web.Streamer.stream("public:media", activity) + + if activity.local do + Pleroma.Web.Streamer.stream("public:local:media", activity) + end + end + end + else + if get_visibility(activity) == "direct", + do: Pleroma.Web.Streamer.stream("direct", activity) + end + end + end end def create(%{to: to, actor: actor, context: context, object: object} = params, fake \\ false) do diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 3c26eb406..dbd3542ea 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.OAuth.Token - alias Pleroma.Web.Streamer @behaviour :cowboy_websocket @@ -25,7 +24,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do ] @anonymous_streams ["public", "public:local", "hashtag"] - # Handled by periodic keepalive in Pleroma.Web.Streamer.Ping. + # Handled by periodic keepalive in Pleroma.Web.Streamer. @timeout :infinity def init(%{qs: qs} = req, state) do @@ -66,7 +65,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do }, topic #{state.topic}" ) - Streamer.add_socket(state.topic, streamer_socket(state)) + Pleroma.Web.Streamer.add_socket(state.topic, streamer_socket(state)) {:ok, state} end @@ -81,7 +80,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do }, topic #{state.topic || "?"}: #{inspect(reason)}" ) - Streamer.remove_socket(state.topic, streamer_socket(state)) + Pleroma.Web.Streamer.remove_socket(state.topic, streamer_socket(state)) :ok end diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex new file mode 100644 index 000000000..587c43f40 --- /dev/null +++ b/lib/pleroma/web/streamer.ex @@ -0,0 +1,318 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Streamer do + use GenServer + require Logger + alias Pleroma.Activity + alias Pleroma.Config + alias Pleroma.Conversation.Participation + alias Pleroma.Notification + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.MastodonAPI.NotificationView + + @keepalive_interval :timer.seconds(30) + + def start_link(_) do + GenServer.start_link(__MODULE__, %{}, name: __MODULE__) + end + + def add_socket(topic, socket) do + GenServer.cast(__MODULE__, %{action: :add, socket: socket, topic: topic}) + end + + def remove_socket(topic, socket) do + GenServer.cast(__MODULE__, %{action: :remove, socket: socket, topic: topic}) + end + + def stream(topic, item) do + GenServer.cast(__MODULE__, %{action: :stream, topic: topic, item: item}) + end + + def init(args) do + Process.send_after(self(), %{action: :ping}, @keepalive_interval) + + {:ok, args} + end + + def handle_info(%{action: :ping}, topics) do + topics + |> Map.values() + |> List.flatten() + |> Enum.each(fn socket -> + Logger.debug("Sending keepalive ping") + send(socket.transport_pid, {:text, ""}) + end) + + Process.send_after(self(), %{action: :ping}, @keepalive_interval) + + {:noreply, topics} + end + + def handle_cast(%{action: :stream, topic: "direct", item: item}, topics) do + recipient_topics = + User.get_recipients_from_activity(item) + |> Enum.map(fn %{id: id} -> "direct:#{id}" end) + + Enum.each(recipient_topics || [], fn user_topic -> + Logger.debug("Trying to push direct message to #{user_topic}\n\n") + push_to_socket(topics, user_topic, item) + end) + + {:noreply, topics} + end + + def handle_cast(%{action: :stream, topic: "participation", item: participation}, topics) do + user_topic = "direct:#{participation.user_id}" + Logger.debug("Trying to push a conversation participation to #{user_topic}\n\n") + + push_to_socket(topics, user_topic, participation) + + {:noreply, topics} + end + + def handle_cast(%{action: :stream, topic: "list", item: item}, topics) do + # filter the recipient list if the activity is not public, see #270. + recipient_lists = + case Visibility.is_public?(item) do + true -> + Pleroma.List.get_lists_from_activity(item) + + _ -> + Pleroma.List.get_lists_from_activity(item) + |> Enum.filter(fn list -> + owner = User.get_cached_by_id(list.user_id) + + Visibility.visible_for_user?(item, owner) + end) + end + + recipient_topics = + recipient_lists + |> Enum.map(fn %{id: id} -> "list:#{id}" end) + + Enum.each(recipient_topics || [], fn list_topic -> + Logger.debug("Trying to push message to #{list_topic}\n\n") + push_to_socket(topics, list_topic, item) + end) + + {:noreply, topics} + end + + def handle_cast( + %{action: :stream, topic: topic, item: %Notification{} = item}, + topics + ) + when topic in ["user", "user:notification"] do + topics + |> Map.get("#{topic}:#{item.user_id}", []) + |> Enum.each(fn socket -> + with %User{} = user <- User.get_cached_by_ap_id(socket.assigns[:user].ap_id), + true <- should_send?(user, item) do + send( + socket.transport_pid, + {:text, represent_notification(socket.assigns[:user], item)} + ) + end + end) + + {:noreply, topics} + end + + def handle_cast(%{action: :stream, topic: "user", item: item}, topics) do + Logger.debug("Trying to push to users") + + recipient_topics = + User.get_recipients_from_activity(item) + |> Enum.map(fn %{id: id} -> "user:#{id}" end) + + Enum.each(recipient_topics, fn topic -> + push_to_socket(topics, topic, item) + end) + + {:noreply, topics} + end + + def handle_cast(%{action: :stream, topic: topic, item: item}, topics) do + Logger.debug("Trying to push to #{topic}") + Logger.debug("Pushing item to #{topic}") + push_to_socket(topics, topic, item) + {:noreply, topics} + end + + def handle_cast(%{action: :add, topic: topic, socket: socket}, sockets) do + topic = internal_topic(topic, socket) + sockets_for_topic = sockets[topic] || [] + sockets_for_topic = Enum.uniq([socket | sockets_for_topic]) + sockets = Map.put(sockets, topic, sockets_for_topic) + Logger.debug("Got new conn for #{topic}") + {:noreply, sockets} + end + + def handle_cast(%{action: :remove, topic: topic, socket: socket}, sockets) do + topic = internal_topic(topic, socket) + sockets_for_topic = sockets[topic] || [] + sockets_for_topic = List.delete(sockets_for_topic, socket) + sockets = Map.put(sockets, topic, sockets_for_topic) + Logger.debug("Removed conn for #{topic}") + {:noreply, sockets} + end + + def handle_cast(m, state) do + Logger.info("Unknown: #{inspect(m)}, #{inspect(state)}") + {:noreply, state} + end + + defp represent_update(%Activity{} = activity, %User{} = user) do + %{ + event: "update", + payload: + Pleroma.Web.MastodonAPI.StatusView.render( + "status.json", + activity: activity, + for: user + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + + defp represent_update(%Activity{} = activity) do + %{ + event: "update", + payload: + Pleroma.Web.MastodonAPI.StatusView.render( + "status.json", + activity: activity + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + + def represent_conversation(%Participation{} = participation) do + %{ + event: "conversation", + payload: + Pleroma.Web.MastodonAPI.ConversationView.render("participation.json", %{ + participation: participation, + for: participation.user + }) + |> Jason.encode!() + } + |> Jason.encode!() + end + + @spec represent_notification(User.t(), Notification.t()) :: binary() + defp represent_notification(%User{} = user, %Notification{} = notify) do + %{ + event: "notification", + payload: + NotificationView.render( + "show.json", + %{notification: notify, for: user} + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + + defp should_send?(%User{} = user, %Activity{} = item) do + blocks = user.info.blocks || [] + mutes = user.info.mutes || [] + reblog_mutes = user.info.muted_reblogs || [] + domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(user.info.domain_blocks) + + with parent when not is_nil(parent) <- Object.normalize(item), + true <- Enum.all?([blocks, mutes, reblog_mutes], &(item.actor not in &1)), + true <- Enum.all?([blocks, mutes], &(parent.data["actor"] not in &1)), + %{host: item_host} <- URI.parse(item.actor), + %{host: parent_host} <- URI.parse(parent.data["actor"]), + false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host), + false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, parent_host), + true <- thread_containment(item, user), + false <- CommonAPI.thread_muted?(user, item) do + true + else + _ -> false + end + end + + defp should_send?(%User{} = user, %Notification{activity: activity}) do + should_send?(user, activity) + end + + def push_to_socket(topics, topic, %Activity{data: %{"type" => "Announce"}} = item) do + Enum.each(topics[topic] || [], fn socket -> + # Get the current user so we have up-to-date blocks etc. + if socket.assigns[:user] do + user = User.get_cached_by_ap_id(socket.assigns[:user].ap_id) + + if should_send?(user, item) do + send(socket.transport_pid, {:text, represent_update(item, user)}) + end + else + send(socket.transport_pid, {:text, represent_update(item)}) + end + end) + end + + def push_to_socket(topics, topic, %Participation{} = participation) do + Enum.each(topics[topic] || [], fn socket -> + send(socket.transport_pid, {:text, represent_conversation(participation)}) + end) + end + + def push_to_socket(topics, topic, %Activity{ + data: %{"type" => "Delete", "deleted_activity_id" => deleted_activity_id} + }) do + Enum.each(topics[topic] || [], fn socket -> + send( + socket.transport_pid, + {:text, %{event: "delete", payload: to_string(deleted_activity_id)} |> Jason.encode!()} + ) + end) + end + + def push_to_socket(_topics, _topic, %Activity{data: %{"type" => "Delete"}}), do: :noop + + def push_to_socket(topics, topic, item) do + Enum.each(topics[topic] || [], fn socket -> + # Get the current user so we have up-to-date blocks etc. + if socket.assigns[:user] do + user = User.get_cached_by_ap_id(socket.assigns[:user].ap_id) + blocks = user.info.blocks || [] + mutes = user.info.mutes || [] + + with true <- Enum.all?([blocks, mutes], &(item.actor not in &1)), + true <- thread_containment(item, user) do + send(socket.transport_pid, {:text, represent_update(item, user)}) + end + else + send(socket.transport_pid, {:text, represent_update(item)}) + end + end) + end + + defp internal_topic(topic, socket) when topic in ~w[user user:notification direct] do + "#{topic}:#{socket.assigns[:user].id}" + end + + defp internal_topic(topic, _), do: topic + + @spec thread_containment(Activity.t(), User.t()) :: boolean() + defp thread_containment(_activity, %User{info: %{skip_thread_containment: true}}), do: true + + defp thread_containment(activity, user) do + if Config.get([:instance, :skip_thread_containment]) do + true + else + ActivityPub.contain_activity(activity, user) + end + end +end diff --git a/lib/pleroma/web/streamer/ping.ex b/lib/pleroma/web/streamer/ping.ex deleted file mode 100644 index f77cbb95c..000000000 --- a/lib/pleroma/web/streamer/ping.ex +++ /dev/null @@ -1,33 +0,0 @@ -defmodule Pleroma.Web.Streamer.Ping do - use GenServer - require Logger - - alias Pleroma.Web.Streamer.State - alias Pleroma.Web.Streamer.StreamerSocket - - @keepalive_interval :timer.seconds(30) - - def start_link(opts) do - ping_interval = Keyword.get(opts, :ping_interval, @keepalive_interval) - GenServer.start_link(__MODULE__, %{ping_interval: ping_interval}, name: __MODULE__) - end - - def init(%{ping_interval: ping_interval} = args) do - Process.send_after(self(), :ping, ping_interval) - {:ok, args} - end - - def handle_info(:ping, %{ping_interval: ping_interval} = state) do - State.get_sockets() - |> Map.values() - |> List.flatten() - |> Enum.each(fn %StreamerSocket{transport_pid: transport_pid} -> - Logger.debug("Sending keepalive ping") - send(transport_pid, {:text, ""}) - end) - - Process.send_after(self(), :ping, ping_interval) - - {:noreply, state} - end -end diff --git a/lib/pleroma/web/streamer/state.ex b/lib/pleroma/web/streamer/state.ex deleted file mode 100644 index 7b5199068..000000000 --- a/lib/pleroma/web/streamer/state.ex +++ /dev/null @@ -1,68 +0,0 @@ -defmodule Pleroma.Web.Streamer.State do - use GenServer - require Logger - - alias Pleroma.Web.Streamer.StreamerSocket - - def start_link(_) do - GenServer.start_link(__MODULE__, %{sockets: %{}}, name: __MODULE__) - end - - def add_socket(topic, socket) do - GenServer.call(__MODULE__, {:add, socket, topic}) - end - - def remove_socket(topic, socket) do - GenServer.call(__MODULE__, {:remove, socket, topic}) - end - - def get_sockets do - %{sockets: stream_sockets} = GenServer.call(__MODULE__, :get_state) - stream_sockets - end - - def init(init_arg) do - {:ok, init_arg} - end - - def handle_call(:get_state, _from, state) do - {:reply, state, state} - end - - def handle_call({:add, socket, topic}, _from, %{sockets: sockets} = state) do - internal_topic = internal_topic(topic, socket) - stream_socket = StreamerSocket.from_socket(socket) - - sockets_for_topic = - sockets - |> Map.get(internal_topic, []) - |> List.insert_at(0, stream_socket) - |> Enum.uniq() - - state = put_in(state, [:sockets, internal_topic], sockets_for_topic) - Logger.debug("Got new conn for #{topic}") - {:reply, state, state} - end - - def handle_call({:remove, socket, topic}, _from, %{sockets: sockets} = state) do - internal_topic = internal_topic(topic, socket) - stream_socket = StreamerSocket.from_socket(socket) - - sockets_for_topic = - sockets - |> Map.get(internal_topic, []) - |> List.delete(stream_socket) - - state = Kernel.put_in(state, [:sockets, internal_topic], sockets_for_topic) - {:reply, state, state} - end - - defp internal_topic(topic, socket) - when topic in ~w[user user:notification direct] do - "#{topic}:#{socket.assigns[:user].id}" - end - - defp internal_topic(topic, _) do - topic - end -end diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex deleted file mode 100644 index 8cf719277..000000000 --- a/lib/pleroma/web/streamer/streamer.ex +++ /dev/null @@ -1,55 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Streamer do - alias Pleroma.Web.Streamer.State - alias Pleroma.Web.Streamer.Worker - - @timeout 60_000 - @mix_env Mix.env() - - def add_socket(topic, socket) do - State.add_socket(topic, socket) - end - - def remove_socket(topic, socket) do - State.remove_socket(topic, socket) - end - - def get_sockets do - State.get_sockets() - end - - def stream(topics, items) do - if should_send?() do - Task.async(fn -> - :poolboy.transaction( - :streamer_worker, - &Worker.stream(&1, topics, items), - @timeout - ) - end) - end - end - - def supervisor, do: Pleroma.Web.Streamer.Supervisor - - defp should_send? do - handle_should_send(@mix_env) - end - - defp handle_should_send(:test) do - case Process.whereis(:streamer_worker) do - nil -> - false - - pid -> - Process.alive?(pid) - end - end - - defp handle_should_send(_) do - true - end -end diff --git a/lib/pleroma/web/streamer/streamer_socket.ex b/lib/pleroma/web/streamer/streamer_socket.ex deleted file mode 100644 index f006c0306..000000000 --- a/lib/pleroma/web/streamer/streamer_socket.ex +++ /dev/null @@ -1,31 +0,0 @@ -defmodule Pleroma.Web.Streamer.StreamerSocket do - defstruct transport_pid: nil, user: nil - - alias Pleroma.User - alias Pleroma.Web.Streamer.StreamerSocket - - def from_socket(%{ - transport_pid: transport_pid, - assigns: %{user: nil} - }) do - %StreamerSocket{ - transport_pid: transport_pid - } - end - - def from_socket(%{ - transport_pid: transport_pid, - assigns: %{user: %User{} = user} - }) do - %StreamerSocket{ - transport_pid: transport_pid, - user: user - } - end - - def from_socket(%{transport_pid: transport_pid}) do - %StreamerSocket{ - transport_pid: transport_pid - } - end -end diff --git a/lib/pleroma/web/streamer/supervisor.ex b/lib/pleroma/web/streamer/supervisor.ex deleted file mode 100644 index 6afe19323..000000000 --- a/lib/pleroma/web/streamer/supervisor.ex +++ /dev/null @@ -1,33 +0,0 @@ -defmodule Pleroma.Web.Streamer.Supervisor do - use Supervisor - - def start_link(opts) do - Supervisor.start_link(__MODULE__, opts, name: __MODULE__) - end - - def init(args) do - children = [ - {Pleroma.Web.Streamer.State, args}, - {Pleroma.Web.Streamer.Ping, args}, - :poolboy.child_spec(:streamer_worker, poolboy_config()) - ] - - opts = [strategy: :one_for_one, name: Pleroma.Web.Streamer.Supervisor] - Supervisor.init(children, opts) - end - - defp poolboy_config do - opts = - Pleroma.Config.get(:streamer, - workers: 3, - overflow_workers: 2 - ) - - [ - {:name, {:local, :streamer_worker}}, - {:worker_module, Pleroma.Web.Streamer.Worker}, - {:size, opts[:workers]}, - {:max_overflow, opts[:overflow_workers]} - ] - end -end diff --git a/lib/pleroma/web/streamer/worker.ex b/lib/pleroma/web/streamer/worker.ex deleted file mode 100644 index 5804508eb..000000000 --- a/lib/pleroma/web/streamer/worker.ex +++ /dev/null @@ -1,220 +0,0 @@ -defmodule Pleroma.Web.Streamer.Worker do - use GenServer - - require Logger - - alias Pleroma.Activity - alias Pleroma.Config - alias Pleroma.Conversation.Participation - alias Pleroma.Notification - alias Pleroma.Object - alias Pleroma.User - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.ActivityPub.Visibility - alias Pleroma.Web.CommonAPI - alias Pleroma.Web.Streamer.State - alias Pleroma.Web.Streamer.StreamerSocket - alias Pleroma.Web.StreamerView - - def start_link(_) do - GenServer.start_link(__MODULE__, %{}, []) - end - - def init(init_arg) do - {:ok, init_arg} - end - - def stream(pid, topics, items) do - GenServer.call(pid, {:stream, topics, items}) - end - - def handle_call({:stream, topics, item}, _from, state) when is_list(topics) do - Enum.each(topics, fn t -> - do_stream(%{topic: t, item: item}) - end) - - {:reply, state, state} - end - - def handle_call({:stream, topic, items}, _from, state) when is_list(items) do - Enum.each(items, fn i -> - do_stream(%{topic: topic, item: i}) - end) - - {:reply, state, state} - end - - def handle_call({:stream, topic, item}, _from, state) do - do_stream(%{topic: topic, item: item}) - - {:reply, state, state} - end - - defp do_stream(%{topic: "direct", item: item}) do - recipient_topics = - User.get_recipients_from_activity(item) - |> Enum.map(fn %{id: id} -> "direct:#{id}" end) - - Enum.each(recipient_topics, fn user_topic -> - Logger.debug("Trying to push direct message to #{user_topic}\n\n") - push_to_socket(State.get_sockets(), user_topic, item) - end) - end - - defp do_stream(%{topic: "participation", item: participation}) do - user_topic = "direct:#{participation.user_id}" - Logger.debug("Trying to push a conversation participation to #{user_topic}\n\n") - - push_to_socket(State.get_sockets(), user_topic, participation) - end - - defp do_stream(%{topic: "list", item: item}) do - # filter the recipient list if the activity is not public, see #270. - recipient_lists = - case Visibility.is_public?(item) do - true -> - Pleroma.List.get_lists_from_activity(item) - - _ -> - Pleroma.List.get_lists_from_activity(item) - |> Enum.filter(fn list -> - owner = User.get_cached_by_id(list.user_id) - - Visibility.visible_for_user?(item, owner) - end) - end - - recipient_topics = - recipient_lists - |> Enum.map(fn %{id: id} -> "list:#{id}" end) - - Enum.each(recipient_topics, fn list_topic -> - Logger.debug("Trying to push message to #{list_topic}\n\n") - push_to_socket(State.get_sockets(), list_topic, item) - end) - end - - defp do_stream(%{topic: topic, item: %Notification{} = item}) - when topic in ["user", "user:notification"] do - State.get_sockets() - |> Map.get("#{topic}:#{item.user_id}", []) - |> Enum.each(fn %StreamerSocket{transport_pid: transport_pid, user: socket_user} -> - with %User{} = user <- User.get_cached_by_ap_id(socket_user.ap_id), - true <- should_send?(user, item) do - send(transport_pid, {:text, StreamerView.render("notification.json", socket_user, item)}) - end - end) - end - - defp do_stream(%{topic: "user", item: item}) do - Logger.debug("Trying to push to users") - - recipient_topics = - User.get_recipients_from_activity(item) - |> Enum.map(fn %{id: id} -> "user:#{id}" end) - - Enum.each(recipient_topics, fn topic -> - push_to_socket(State.get_sockets(), topic, item) - end) - end - - defp do_stream(%{topic: topic, item: item}) do - Logger.debug("Trying to push to #{topic}") - Logger.debug("Pushing item to #{topic}") - push_to_socket(State.get_sockets(), topic, item) - end - - defp should_send?(%User{} = user, %Activity{} = item) do - blocks = user.info.blocks || [] - mutes = user.info.mutes || [] - reblog_mutes = user.info.muted_reblogs || [] - domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(user.info.domain_blocks) - - with parent when not is_nil(parent) <- Object.normalize(item), - true <- Enum.all?([blocks, mutes, reblog_mutes], &(item.actor not in &1)), - true <- Enum.all?([blocks, mutes], &(parent.data["actor"] not in &1)), - %{host: item_host} <- URI.parse(item.actor), - %{host: parent_host} <- URI.parse(parent.data["actor"]), - false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host), - false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, parent_host), - true <- thread_containment(item, user), - false <- CommonAPI.thread_muted?(user, item) do - true - else - _ -> false - end - end - - defp should_send?(%User{} = user, %Notification{activity: activity}) do - should_send?(user, activity) - end - - def push_to_socket(topics, topic, %Activity{data: %{"type" => "Announce"}} = item) do - Enum.each(topics[topic] || [], fn %StreamerSocket{ - transport_pid: transport_pid, - user: socket_user - } -> - # Get the current user so we have up-to-date blocks etc. - if socket_user do - user = User.get_cached_by_ap_id(socket_user.ap_id) - - if should_send?(user, item) do - send(transport_pid, {:text, StreamerView.render("update.json", item, user)}) - end - else - send(transport_pid, {:text, StreamerView.render("update.json", item)}) - end - end) - end - - def push_to_socket(topics, topic, %Participation{} = participation) do - Enum.each(topics[topic] || [], fn %StreamerSocket{transport_pid: transport_pid} -> - send(transport_pid, {:text, StreamerView.render("conversation.json", participation)}) - end) - end - - def push_to_socket(topics, topic, %Activity{ - data: %{"type" => "Delete", "deleted_activity_id" => deleted_activity_id} - }) do - Enum.each(topics[topic] || [], fn %StreamerSocket{transport_pid: transport_pid} -> - send( - transport_pid, - {:text, %{event: "delete", payload: to_string(deleted_activity_id)} |> Jason.encode!()} - ) - end) - end - - def push_to_socket(_topics, _topic, %Activity{data: %{"type" => "Delete"}}), do: :noop - - def push_to_socket(topics, topic, item) do - Enum.each(topics[topic] || [], fn %StreamerSocket{ - transport_pid: transport_pid, - user: socket_user - } -> - # Get the current user so we have up-to-date blocks etc. - if socket_user do - user = User.get_cached_by_ap_id(socket_user.ap_id) - blocks = user.info.blocks || [] - mutes = user.info.mutes || [] - - with true <- Enum.all?([blocks, mutes], &(item.actor not in &1)), - true <- thread_containment(item, user) do - send(transport_pid, {:text, StreamerView.render("update.json", item, user)}) - end - else - send(transport_pid, {:text, StreamerView.render("update.json", item)}) - end - end) - end - - @spec thread_containment(Activity.t(), User.t()) :: boolean() - defp thread_containment(_activity, %User{info: %{skip_thread_containment: true}}), do: true - - defp thread_containment(activity, user) do - if Config.get([:instance, :skip_thread_containment]) do - true - else - ActivityPub.contain_activity(activity, user) - end - end -end diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex deleted file mode 100644 index b13030fa0..000000000 --- a/lib/pleroma/web/views/streamer_view.ex +++ /dev/null @@ -1,66 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.StreamerView do - use Pleroma.Web, :view - - alias Pleroma.Activity - alias Pleroma.Conversation.Participation - alias Pleroma.Notification - alias Pleroma.User - alias Pleroma.Web.MastodonAPI.NotificationView - - def render("update.json", %Activity{} = activity, %User{} = user) do - %{ - event: "update", - payload: - Pleroma.Web.MastodonAPI.StatusView.render( - "status.json", - activity: activity, - for: user - ) - |> Jason.encode!() - } - |> Jason.encode!() - end - - def render("notification.json", %User{} = user, %Notification{} = notify) do - %{ - event: "notification", - payload: - NotificationView.render( - "show.json", - %{notification: notify, for: user} - ) - |> Jason.encode!() - } - |> Jason.encode!() - end - - def render("update.json", %Activity{} = activity) do - %{ - event: "update", - payload: - Pleroma.Web.MastodonAPI.StatusView.render( - "status.json", - activity: activity - ) - |> Jason.encode!() - } - |> Jason.encode!() - end - - def render("conversation.json", %Participation{} = participation) do - %{ - event: "conversation", - payload: - Pleroma.Web.MastodonAPI.ConversationView.render("participation.json", %{ - participation: participation, - for: participation.user - }) - |> Jason.encode!() - } - |> Jason.encode!() - end -end diff --git a/mix.exs b/mix.exs index 911ebad1d..f1e98585b 100644 --- a/mix.exs +++ b/mix.exs @@ -144,7 +144,6 @@ defmodule Pleroma.Mixfile do git: "https://git.pleroma.social/pleroma/http_signatures.git", ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"}, {:telemetry, "~> 0.3"}, - {:poolboy, "~> 1.5"}, {:prometheus_ex, "~> 3.0"}, {:prometheus_plugs, "~> 1.1"}, {:prometheus_phoenix, "~> 1.3"}, diff --git a/mix.lock b/mix.lock index 0bf6a811e..41697dd5c 100644 --- a/mix.lock +++ b/mix.lock @@ -73,7 +73,6 @@ "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"}, "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, - "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "prometheus": {:hex, :prometheus, "4.4.1", "1e96073b3ed7788053768fea779cbc896ddc3bdd9ba60687f2ad50b252ac87d6", [:mix, :rebar3], [], "hexpm"}, "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/activity/ir/topics_test.exs b/test/activity/ir/topics_test.exs deleted file mode 100644 index e75f83586..000000000 --- a/test/activity/ir/topics_test.exs +++ /dev/null @@ -1,141 +0,0 @@ -defmodule Pleroma.Activity.Ir.TopicsTest do - use Pleroma.DataCase - - alias Pleroma.Activity - alias Pleroma.Activity.Ir.Topics - alias Pleroma.Object - - require Pleroma.Constants - - describe "poll answer" do - test "produce no topics" do - activity = %Activity{object: %Object{data: %{"type" => "Answer"}}} - - assert [] == Topics.get_activity_topics(activity) - end - end - - describe "non poll answer" do - test "always add user and list topics" do - activity = %Activity{object: %Object{data: %{"type" => "FooBar"}}} - topics = Topics.get_activity_topics(activity) - - assert Enum.member?(topics, "user") - assert Enum.member?(topics, "list") - end - end - - describe "public visibility" do - setup do - activity = %Activity{ - object: %Object{data: %{"type" => "Note"}}, - data: %{"to" => [Pleroma.Constants.as_public()]} - } - - {:ok, activity: activity} - end - - test "produces public topic", %{activity: activity} do - topics = Topics.get_activity_topics(activity) - - assert Enum.member?(topics, "public") - end - - test "local action produces public:local topic", %{activity: activity} do - activity = %{activity | local: true} - topics = Topics.get_activity_topics(activity) - - assert Enum.member?(topics, "public:local") - end - - test "non-local action does not produce public:local topic", %{activity: activity} do - activity = %{activity | local: false} - topics = Topics.get_activity_topics(activity) - - refute Enum.member?(topics, "public:local") - end - end - - describe "public visibility create events" do - setup do - activity = %Activity{ - object: %Object{data: %{"type" => "Create", "attachment" => []}}, - data: %{"to" => [Pleroma.Constants.as_public()]} - } - - {:ok, activity: activity} - end - - test "with no attachments doesn't produce public:media topics", %{activity: activity} do - topics = Topics.get_activity_topics(activity) - - refute Enum.member?(topics, "public:media") - refute Enum.member?(topics, "public:local:media") - end - - test "converts tags to hash tags", %{activity: %{object: %{data: data} = object} = activity} do - tagged_data = Map.put(data, "tag", ["foo", "bar"]) - activity = %{activity | object: %{object | data: tagged_data}} - - topics = Topics.get_activity_topics(activity) - - assert Enum.member?(topics, "hashtag:foo") - assert Enum.member?(topics, "hashtag:bar") - end - - test "only converts strinngs to hash tags", %{ - activity: %{object: %{data: data} = object} = activity - } do - tagged_data = Map.put(data, "tag", [2]) - activity = %{activity | object: %{object | data: tagged_data}} - - topics = Topics.get_activity_topics(activity) - - refute Enum.member?(topics, "hashtag:2") - end - end - - describe "public visibility create events with attachments" do - setup do - activity = %Activity{ - object: %Object{data: %{"type" => "Create", "attachment" => ["foo"]}}, - data: %{"to" => [Pleroma.Constants.as_public()]} - } - - {:ok, activity: activity} - end - - test "produce public:media topics", %{activity: activity} do - topics = Topics.get_activity_topics(activity) - - assert Enum.member?(topics, "public:media") - end - - test "local produces public:local:media topics", %{activity: activity} do - topics = Topics.get_activity_topics(activity) - - assert Enum.member?(topics, "public:local:media") - end - - test "non-local doesn't produce public:local:media topics", %{activity: activity} do - activity = %{activity | local: false} - - topics = Topics.get_activity_topics(activity) - - refute Enum.member?(topics, "public:local:media") - end - end - - describe "non-public visibility" do - test "produces direct topic" do - activity = %Activity{object: %Object{data: %{"type" => "Note"}}, data: %{"to" => []}} - topics = Topics.get_activity_topics(activity) - - assert Enum.member?(topics, "direct") - refute Enum.member?(topics, "public") - refute Enum.member?(topics, "public:local") - refute Enum.member?(topics, "public:media") - refute Enum.member?(topics, "public:local:media") - end - end -end diff --git a/test/integration/mastodon_websocket_test.exs b/test/integration/mastodon_websocket_test.exs index c04262808..63bf73412 100644 --- a/test/integration/mastodon_websocket_test.exs +++ b/test/integration/mastodon_websocket_test.exs @@ -11,6 +11,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do alias Pleroma.Integration.WebsocketClient alias Pleroma.Web.CommonAPI alias Pleroma.Web.OAuth + alias Pleroma.Web.Streamer @path Pleroma.Web.Endpoint.url() |> URI.parse() @@ -18,6 +19,16 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do |> Map.put(:path, "/api/v1/streaming") |> URI.to_string() + setup do + GenServer.start(Streamer, %{}, name: Streamer) + + on_exit(fn -> + if pid = Process.whereis(Streamer) do + Process.exit(pid, :kill) + end + end) + end + def start_socket(qs \\ nil, headers \\ []) do path = case qs do @@ -42,14 +53,12 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) end - @tag needs_streamer: true test "allows public streams without authentication" do assert {:ok, _} = start_socket("?stream=public") assert {:ok, _} = start_socket("?stream=public:local") assert {:ok, _} = start_socket("?stream=hashtag&tag=lain") end - @tag needs_streamer: true test "receives well formatted events" do user = insert(:user) {:ok, _} = start_socket("?stream=public") @@ -94,7 +103,6 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert {:ok, _} = start_socket("?stream=user&access_token=#{state.token.token}") end - @tag needs_streamer: true test "accepts the 'user' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user&access_token=#{token.token}") @@ -103,7 +111,6 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) =~ ":badarg" end - @tag needs_streamer: true test "accepts the 'user:notification' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user:notification&access_token=#{token.token}") @@ -112,7 +119,6 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) =~ ":badarg" end - @tag needs_streamer: true test "accepts valid token on Sec-WebSocket-Protocol header", %{token: token} do assert {:ok, _} = start_socket("?stream=user", [{"Sec-WebSocket-Protocol", token.token}]) diff --git a/test/notification_test.exs b/test/notification_test.exs index 3d2f9a8fc..3be9db09b 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -69,7 +69,16 @@ defmodule Pleroma.NotificationTest do end describe "create_notification" do - @tag needs_streamer: true + setup do + GenServer.start(Streamer, %{}, name: Streamer) + + on_exit(fn -> + if pid = Process.whereis(Streamer) do + Process.exit(pid, :kill) + end + end) + end + test "it creates a notification for user and send to the 'user' and the 'user:notification' stream" do user = insert(:user) task = Task.async(fn -> assert_receive {:text, _}, 4_000 end) diff --git a/test/support/conn_case.ex b/test/support/conn_case.ex index b39c70677..ec5892ff5 100644 --- a/test/support/conn_case.ex +++ b/test/support/conn_case.ex @@ -40,10 +40,6 @@ defmodule Pleroma.Web.ConnCase do Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, {:shared, self()}) end - if tags[:needs_streamer] do - start_supervised(Pleroma.Web.Streamer.supervisor()) - end - {:ok, conn: Phoenix.ConnTest.build_conn()} end end diff --git a/test/support/data_case.ex b/test/support/data_case.ex index 17fa15214..f3d98e7e3 100644 --- a/test/support/data_case.ex +++ b/test/support/data_case.ex @@ -39,10 +39,6 @@ defmodule Pleroma.DataCase do Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, {:shared, self()}) end - if tags[:needs_streamer] do - start_supervised(Pleroma.Web.Streamer.supervisor()) - end - :ok end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 4100108a5..d0118fefa 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -38,7 +38,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do stream: fn _, _ -> nil end do ActivityPub.stream_out_participations(conversation.participations) - assert called(Pleroma.Web.Streamer.stream("participation", participations)) + Enum.each(participations, fn participation -> + assert called(Pleroma.Web.Streamer.stream("participation", participation)) + end) end end end diff --git a/test/web/streamer/ping_test.exs b/test/web/streamer/ping_test.exs deleted file mode 100644 index 3d52c00e4..000000000 --- a/test/web/streamer/ping_test.exs +++ /dev/null @@ -1,36 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.PingTest do - use Pleroma.DataCase - - import Pleroma.Factory - alias Pleroma.Web.Streamer - - setup do - start_supervised({Streamer.supervisor(), [ping_interval: 30]}) - - :ok - end - - describe "sockets" do - setup do - user = insert(:user) - {:ok, %{user: user}} - end - - test "it sends pings", %{user: user} do - task = - Task.async(fn -> - assert_receive {:text, received_event}, 40 - assert_receive {:text, received_event}, 40 - assert_receive {:text, received_event}, 40 - end) - - Streamer.add_socket("public", %{transport_pid: task.pid, assigns: %{user: user}}) - - Task.await(task) - end - end -end diff --git a/test/web/streamer/state_test.exs b/test/web/streamer/state_test.exs deleted file mode 100644 index d1aeac541..000000000 --- a/test/web/streamer/state_test.exs +++ /dev/null @@ -1,54 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.StateTest do - use Pleroma.DataCase - - import Pleroma.Factory - alias Pleroma.Web.Streamer - alias Pleroma.Web.Streamer.StreamerSocket - - @moduletag needs_streamer: true - - describe "sockets" do - setup do - user = insert(:user) - user2 = insert(:user) - {:ok, %{user: user, user2: user2}} - end - - test "it can add a socket", %{user: user} do - Streamer.add_socket("public", %{transport_pid: 1, assigns: %{user: user}}) - - assert(%{"public" => [%StreamerSocket{transport_pid: 1}]} = Streamer.get_sockets()) - end - - test "it can add multiple sockets per user", %{user: user} do - Streamer.add_socket("public", %{transport_pid: 1, assigns: %{user: user}}) - Streamer.add_socket("public", %{transport_pid: 2, assigns: %{user: user}}) - - assert( - %{ - "public" => [ - %StreamerSocket{transport_pid: 2}, - %StreamerSocket{transport_pid: 1} - ] - } = Streamer.get_sockets() - ) - end - - test "it will not add a duplicate socket", %{user: user} do - Streamer.add_socket("activity", %{transport_pid: 1, assigns: %{user: user}}) - Streamer.add_socket("activity", %{transport_pid: 1, assigns: %{user: user}}) - - assert( - %{ - "activity" => [ - %StreamerSocket{transport_pid: 1} - ] - } = Streamer.get_sockets() - ) - end - end -end diff --git a/test/web/streamer/streamer_test.exs b/test/web/streamer_test.exs similarity index 86% rename from test/web/streamer/streamer_test.exs rename to test/web/streamer_test.exs index 88847e20f..96fa7645f 100644 --- a/test/web/streamer/streamer_test.exs +++ b/test/web/streamer_test.exs @@ -5,20 +5,24 @@ defmodule Pleroma.Web.StreamerTest do use Pleroma.DataCase - import Pleroma.Factory - alias Pleroma.List alias Pleroma.User alias Pleroma.Web.CommonAPI alias Pleroma.Web.Streamer - alias Pleroma.Web.Streamer.StreamerSocket - alias Pleroma.Web.Streamer.Worker + import Pleroma.Factory - @moduletag needs_streamer: true clear_config_all([:instance, :skip_thread_containment]) describe "user streams" do setup do + GenServer.start(Streamer, %{}, name: Streamer) + + on_exit(fn -> + if pid = Process.whereis(Streamer) do + Process.exit(pid, :kill) + end + end) + user = insert(:user) notify = insert(:notification, user: user, activity: build(:note_activity)) {:ok, %{user: user, notify: notify}} @@ -121,9 +125,11 @@ defmodule Pleroma.Web.StreamerTest do assert_receive {:text, _}, 4_000 end) - fake_socket = %StreamerSocket{ + fake_socket = %{ transport_pid: task.pid, - user: user + assigns: %{ + user: user + } } {:ok, activity} = CommonAPI.post(other_user, %{"status" => "Test"}) @@ -132,7 +138,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Worker.push_to_socket(topics, "public", activity) + Streamer.push_to_socket(topics, "public", activity) Task.await(task) @@ -149,9 +155,11 @@ defmodule Pleroma.Web.StreamerTest do assert received_event == expected_event end) - fake_socket = %StreamerSocket{ + fake_socket = %{ transport_pid: task.pid, - user: user + assigns: %{ + user: user + } } {:ok, activity} = CommonAPI.delete(activity.id, other_user) @@ -160,7 +168,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Worker.push_to_socket(topics, "public", activity) + Streamer.push_to_socket(topics, "public", activity) Task.await(task) end @@ -181,9 +189,9 @@ defmodule Pleroma.Web.StreamerTest do ) task = Task.async(fn -> refute_receive {:text, _}, 1_000 end) - fake_socket = %StreamerSocket{transport_pid: task.pid, user: user} + fake_socket = %{transport_pid: task.pid, assigns: %{user: user}} topics = %{"public" => [fake_socket]} - Worker.push_to_socket(topics, "public", activity) + Streamer.push_to_socket(topics, "public", activity) Task.await(task) end @@ -203,9 +211,9 @@ defmodule Pleroma.Web.StreamerTest do ) task = Task.async(fn -> assert_receive {:text, _}, 1_000 end) - fake_socket = %StreamerSocket{transport_pid: task.pid, user: user} + fake_socket = %{transport_pid: task.pid, assigns: %{user: user}} topics = %{"public" => [fake_socket]} - Worker.push_to_socket(topics, "public", activity) + Streamer.push_to_socket(topics, "public", activity) Task.await(task) end @@ -225,9 +233,9 @@ defmodule Pleroma.Web.StreamerTest do ) task = Task.async(fn -> assert_receive {:text, _}, 1_000 end) - fake_socket = %StreamerSocket{transport_pid: task.pid, user: user} + fake_socket = %{transport_pid: task.pid, assigns: %{user: user}} topics = %{"public" => [fake_socket]} - Worker.push_to_socket(topics, "public", activity) + Streamer.push_to_socket(topics, "public", activity) Task.await(task) end @@ -243,9 +251,11 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %StreamerSocket{ + fake_socket = %{ transport_pid: task.pid, - user: user + assigns: %{ + user: user + } } {:ok, activity} = CommonAPI.post(blocked_user, %{"status" => "Test"}) @@ -254,7 +264,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Worker.push_to_socket(topics, "public", activity) + Streamer.push_to_socket(topics, "public", activity) Task.await(task) end @@ -274,9 +284,11 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %StreamerSocket{ + fake_socket = %{ transport_pid: task.pid, - user: user_a + assigns: %{ + user: user_a + } } {:ok, activity} = @@ -289,7 +301,7 @@ defmodule Pleroma.Web.StreamerTest do "list:#{list.id}" => [fake_socket] } - Worker.handle_call({:stream, "list", activity}, self(), topics) + Streamer.handle_cast(%{action: :stream, topic: "list", item: activity}, topics) Task.await(task) end @@ -306,9 +318,11 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %StreamerSocket{ + fake_socket = %{ transport_pid: task.pid, - user: user_a + assigns: %{ + user: user_a + } } {:ok, activity} = @@ -321,12 +335,12 @@ defmodule Pleroma.Web.StreamerTest do "list:#{list.id}" => [fake_socket] } - Worker.handle_call({:stream, "list", activity}, self(), topics) + Streamer.handle_cast(%{action: :stream, topic: "list", item: activity}, topics) Task.await(task) end - test "it sends wanted private posts to list" do + test "it send wanted private posts to list" do user_a = insert(:user) user_b = insert(:user) @@ -340,9 +354,11 @@ defmodule Pleroma.Web.StreamerTest do assert_receive {:text, _}, 1_000 end) - fake_socket = %StreamerSocket{ + fake_socket = %{ transport_pid: task.pid, - user: user_a + assigns: %{ + user: user_a + } } {:ok, activity} = @@ -351,12 +367,11 @@ defmodule Pleroma.Web.StreamerTest do "visibility" => "private" }) - Streamer.add_socket( - "list:#{list.id}", - fake_socket - ) + topics = %{ + "list:#{list.id}" => [fake_socket] + } - Worker.handle_call({:stream, "list", activity}, self(), %{}) + Streamer.handle_cast(%{action: :stream, topic: "list", item: activity}, topics) Task.await(task) end @@ -372,9 +387,11 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %StreamerSocket{ + fake_socket = %{ transport_pid: task.pid, - user: user1 + assigns: %{ + user: user1 + } } {:ok, create_activity} = CommonAPI.post(user3, %{"status" => "I'm kawen"}) @@ -384,7 +401,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Worker.push_to_socket(topics, "public", announce_activity) + Streamer.push_to_socket(topics, "public", announce_activity) Task.await(task) end @@ -400,8 +417,6 @@ defmodule Pleroma.Web.StreamerTest do task = Task.async(fn -> refute_receive {:text, _}, 4_000 end) - Process.sleep(4000) - Streamer.add_socket( "user", %{transport_pid: task.pid, assigns: %{user: user2}} @@ -413,6 +428,14 @@ defmodule Pleroma.Web.StreamerTest do describe "direct streams" do setup do + GenServer.start(Streamer, %{}, name: Streamer) + + on_exit(fn -> + if pid = Process.whereis(Streamer) do + Process.exit(pid, :kill) + end + end) + :ok end @@ -457,8 +480,6 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 4_000 end) - Process.sleep(1000) - Streamer.add_socket( "direct", %{transport_pid: task.pid, assigns: %{user: user}} @@ -500,8 +521,6 @@ defmodule Pleroma.Web.StreamerTest do assert last_status["id"] == to_string(create_activity.id) end) - Process.sleep(1000) - Streamer.add_socket( "direct", %{transport_pid: task.pid, assigns: %{user: user}} From e8120944d8c016a1aa8fcefe34b1f0cc9089ea4f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 16 Sep 2019 13:23:06 +0300 Subject: [PATCH 38/45] Fix signed fetch inclusion publisher test Oban branch changed `actor` to `actor_id` and this test was not adjusted for that --- test/web/activity_pub/publisher_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/web/activity_pub/publisher_test.exs b/test/web/activity_pub/publisher_test.exs index c7d1d05aa..df03b4008 100644 --- a/test/web/activity_pub/publisher_test.exs +++ b/test/web/activity_pub/publisher_test.exs @@ -321,7 +321,7 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do assert called( Pleroma.Web.Federator.Publisher.enqueue_one(Publisher, %{ inbox: "https://domain.com/users/nick1/inbox", - actor: actor, + actor_id: actor.id, id: delete.data["id"] }) ) @@ -329,7 +329,7 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do assert called( Pleroma.Web.Federator.Publisher.enqueue_one(Publisher, %{ inbox: "https://domain2.com/users/nick1/inbox", - actor: actor, + actor_id: actor.id, id: delete.data["id"] }) ) From 96816ceaa25c21cec7677e75dcddd7ffb42d83c3 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 16 Sep 2019 17:03:37 +0700 Subject: [PATCH 39/45] Revert "Merge branch 'revert-4fabf83a' into 'develop'" This reverts commit fe7fd331263007e0fb2877ef7370a09a9704da36, reversing changes made to 4fabf83ad01352442906d79187aeab4c777f4df8. --- .gitignore | 4 + config/config.exs | 4 + lib/pleroma/activity/ir/topics.ex | 63 ++++ lib/pleroma/application.ex | 2 +- lib/pleroma/notification.ex | 6 +- lib/pleroma/web/activity_pub/activity_pub.ex | 48 +-- .../web/mastodon_api/websocket_handler.ex | 7 +- lib/pleroma/web/streamer.ex | 318 ------------------ lib/pleroma/web/streamer/ping.ex | 33 ++ lib/pleroma/web/streamer/state.ex | 68 ++++ lib/pleroma/web/streamer/streamer.ex | 55 +++ lib/pleroma/web/streamer/streamer_socket.ex | 31 ++ lib/pleroma/web/streamer/supervisor.ex | 33 ++ lib/pleroma/web/streamer/worker.ex | 220 ++++++++++++ lib/pleroma/web/views/streamer_view.ex | 66 ++++ mix.exs | 1 + mix.lock | 1 + test/activity/ir/topics_test.exs | 141 ++++++++ test/integration/mastodon_websocket_test.exs | 16 +- test/notification_test.exs | 11 +- test/support/conn_case.ex | 4 + test/support/data_case.ex | 4 + test/web/activity_pub/activity_pub_test.exs | 4 +- test/web/streamer/ping_test.exs | 36 ++ test/web/streamer/state_test.exs | 54 +++ test/web/{ => streamer}/streamer_test.exs | 105 +++--- 26 files changed, 888 insertions(+), 447 deletions(-) create mode 100644 lib/pleroma/activity/ir/topics.ex delete mode 100644 lib/pleroma/web/streamer.ex create mode 100644 lib/pleroma/web/streamer/ping.ex create mode 100644 lib/pleroma/web/streamer/state.ex create mode 100644 lib/pleroma/web/streamer/streamer.ex create mode 100644 lib/pleroma/web/streamer/streamer_socket.ex create mode 100644 lib/pleroma/web/streamer/supervisor.ex create mode 100644 lib/pleroma/web/streamer/worker.ex create mode 100644 lib/pleroma/web/views/streamer_view.ex create mode 100644 test/activity/ir/topics_test.exs create mode 100644 test/web/streamer/ping_test.exs create mode 100644 test/web/streamer/state_test.exs rename test/web/{ => streamer}/streamer_test.exs (86%) diff --git a/.gitignore b/.gitignore index 4e71a7df0..3b0c7d361 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,7 @@ docs/generated_config.md # Code test coverage /cover /Elixir.*.coverdata + +.idea +pleroma.iml + diff --git a/config/config.exs b/config/config.exs index ab6e00c98..b1b98af93 100644 --- a/config/config.exs +++ b/config/config.exs @@ -331,6 +331,10 @@ config :pleroma, :activitypub, follow_handshake_timeout: 500, sign_object_fetches: true +config :pleroma, :streamer, + workers: 3, + overflow_workers: 2 + config :pleroma, :user, deny_follow_blocked: true config :pleroma, :mrf_normalize_markup, scrub_policy: Pleroma.HTML.Scrubber.Default diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex new file mode 100644 index 000000000..010897abc --- /dev/null +++ b/lib/pleroma/activity/ir/topics.ex @@ -0,0 +1,63 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Activity.Ir.Topics do + alias Pleroma.Object + alias Pleroma.Web.ActivityPub.Visibility + + def get_activity_topics(activity) do + activity + |> Object.normalize() + |> generate_topics(activity) + |> List.flatten() + end + + defp generate_topics(%{data: %{"type" => "Answer"}}, _) do + [] + end + + defp generate_topics(object, activity) do + ["user", "list"] ++ visibility_tags(object, activity) + end + + defp visibility_tags(object, activity) do + case Visibility.get_visibility(activity) do + "public" -> + if activity.local do + ["public", "public:local"] + else + ["public"] + end + |> item_creation_tags(object, activity) + + "direct" -> + ["direct"] + + _ -> + [] + end + end + + defp item_creation_tags(tags, %{data: %{"type" => "Create"}} = object, activity) do + tags ++ hashtags_to_topics(object) ++ attachment_topics(object, activity) + end + + defp item_creation_tags(tags, _, _) do + tags + end + + defp hashtags_to_topics(%{data: %{"tag" => tags}}) do + tags + |> Enum.filter(&is_bitstring(&1)) + |> Enum.map(fn tag -> "hashtag:" <> tag end) + end + + defp hashtags_to_topics(_), do: [] + + defp attachment_topics(%{data: %{"attachment" => []}}, _act), do: [] + + defp attachment_topics(_object, %{local: true}), do: ["public:media", "public:local:media"] + + defp attachment_topics(_object, _act), do: ["public:media"] +end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 49094704b..3b37ce630 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -141,7 +141,7 @@ defmodule Pleroma.Application do defp streamer_child(:test), do: [] defp streamer_child(_) do - [Pleroma.Web.Streamer] + [Pleroma.Web.Streamer.supervisor()] end defp oauth_cleanup_child(true), diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index b7c880c51..8012389ac 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -210,8 +210,10 @@ defmodule Pleroma.Notification do unless skip?(activity, user) do notification = %Notification{user_id: user.id, activity: activity} {:ok, notification} = Repo.insert(notification) - Streamer.stream("user", notification) - Streamer.stream("user:notification", notification) + + ["user", "user:notification"] + |> Streamer.stream(notification) + Push.send(notification) notification end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 41f6a0f1f..bc5ae7fbf 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Activity + alias Pleroma.Activity.Ir.Topics alias Pleroma.Config alias Pleroma.Conversation alias Pleroma.Notification @@ -16,6 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.User alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.Streamer alias Pleroma.Web.WebFinger alias Pleroma.Workers.BackgroundWorker @@ -187,9 +189,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do participations |> Repo.preload(:user) - Enum.each(participations, fn participation -> - Pleroma.Web.Streamer.stream("participation", participation) - end) + Streamer.stream("participation", participations) end def stream_out_participations(%Object{data: %{"context" => context}}, user) do @@ -208,41 +208,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def stream_out_participations(_, _), do: :noop - def stream_out(activity) do - if activity.data["type"] in ["Create", "Announce", "Delete"] do - object = Object.normalize(activity) - # Do not stream out poll replies - unless object.data["type"] == "Answer" do - Pleroma.Web.Streamer.stream("user", activity) - Pleroma.Web.Streamer.stream("list", activity) + def stream_out(%Activity{data: %{"type" => data_type}} = activity) + when data_type in ["Create", "Announce", "Delete"] do + activity + |> Topics.get_activity_topics() + |> Streamer.stream(activity) + end - if get_visibility(activity) == "public" do - Pleroma.Web.Streamer.stream("public", activity) - - if activity.local do - Pleroma.Web.Streamer.stream("public:local", activity) - end - - if activity.data["type"] in ["Create"] do - object.data - |> Map.get("tag", []) - |> Enum.filter(fn tag -> is_bitstring(tag) end) - |> Enum.each(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end) - - if object.data["attachment"] != [] do - Pleroma.Web.Streamer.stream("public:media", activity) - - if activity.local do - Pleroma.Web.Streamer.stream("public:local:media", activity) - end - end - end - else - if get_visibility(activity) == "direct", - do: Pleroma.Web.Streamer.stream("direct", activity) - end - end - end + def stream_out(_activity) do + :noop end def create(%{to: to, actor: actor, context: context, object: object} = params, fake \\ false) do diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index dbd3542ea..3c26eb406 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.OAuth.Token + alias Pleroma.Web.Streamer @behaviour :cowboy_websocket @@ -24,7 +25,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do ] @anonymous_streams ["public", "public:local", "hashtag"] - # Handled by periodic keepalive in Pleroma.Web.Streamer. + # Handled by periodic keepalive in Pleroma.Web.Streamer.Ping. @timeout :infinity def init(%{qs: qs} = req, state) do @@ -65,7 +66,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do }, topic #{state.topic}" ) - Pleroma.Web.Streamer.add_socket(state.topic, streamer_socket(state)) + Streamer.add_socket(state.topic, streamer_socket(state)) {:ok, state} end @@ -80,7 +81,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do }, topic #{state.topic || "?"}: #{inspect(reason)}" ) - Pleroma.Web.Streamer.remove_socket(state.topic, streamer_socket(state)) + Streamer.remove_socket(state.topic, streamer_socket(state)) :ok end diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex deleted file mode 100644 index 587c43f40..000000000 --- a/lib/pleroma/web/streamer.ex +++ /dev/null @@ -1,318 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Streamer do - use GenServer - require Logger - alias Pleroma.Activity - alias Pleroma.Config - alias Pleroma.Conversation.Participation - alias Pleroma.Notification - alias Pleroma.Object - alias Pleroma.User - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.ActivityPub.Visibility - alias Pleroma.Web.CommonAPI - alias Pleroma.Web.MastodonAPI.NotificationView - - @keepalive_interval :timer.seconds(30) - - def start_link(_) do - GenServer.start_link(__MODULE__, %{}, name: __MODULE__) - end - - def add_socket(topic, socket) do - GenServer.cast(__MODULE__, %{action: :add, socket: socket, topic: topic}) - end - - def remove_socket(topic, socket) do - GenServer.cast(__MODULE__, %{action: :remove, socket: socket, topic: topic}) - end - - def stream(topic, item) do - GenServer.cast(__MODULE__, %{action: :stream, topic: topic, item: item}) - end - - def init(args) do - Process.send_after(self(), %{action: :ping}, @keepalive_interval) - - {:ok, args} - end - - def handle_info(%{action: :ping}, topics) do - topics - |> Map.values() - |> List.flatten() - |> Enum.each(fn socket -> - Logger.debug("Sending keepalive ping") - send(socket.transport_pid, {:text, ""}) - end) - - Process.send_after(self(), %{action: :ping}, @keepalive_interval) - - {:noreply, topics} - end - - def handle_cast(%{action: :stream, topic: "direct", item: item}, topics) do - recipient_topics = - User.get_recipients_from_activity(item) - |> Enum.map(fn %{id: id} -> "direct:#{id}" end) - - Enum.each(recipient_topics || [], fn user_topic -> - Logger.debug("Trying to push direct message to #{user_topic}\n\n") - push_to_socket(topics, user_topic, item) - end) - - {:noreply, topics} - end - - def handle_cast(%{action: :stream, topic: "participation", item: participation}, topics) do - user_topic = "direct:#{participation.user_id}" - Logger.debug("Trying to push a conversation participation to #{user_topic}\n\n") - - push_to_socket(topics, user_topic, participation) - - {:noreply, topics} - end - - def handle_cast(%{action: :stream, topic: "list", item: item}, topics) do - # filter the recipient list if the activity is not public, see #270. - recipient_lists = - case Visibility.is_public?(item) do - true -> - Pleroma.List.get_lists_from_activity(item) - - _ -> - Pleroma.List.get_lists_from_activity(item) - |> Enum.filter(fn list -> - owner = User.get_cached_by_id(list.user_id) - - Visibility.visible_for_user?(item, owner) - end) - end - - recipient_topics = - recipient_lists - |> Enum.map(fn %{id: id} -> "list:#{id}" end) - - Enum.each(recipient_topics || [], fn list_topic -> - Logger.debug("Trying to push message to #{list_topic}\n\n") - push_to_socket(topics, list_topic, item) - end) - - {:noreply, topics} - end - - def handle_cast( - %{action: :stream, topic: topic, item: %Notification{} = item}, - topics - ) - when topic in ["user", "user:notification"] do - topics - |> Map.get("#{topic}:#{item.user_id}", []) - |> Enum.each(fn socket -> - with %User{} = user <- User.get_cached_by_ap_id(socket.assigns[:user].ap_id), - true <- should_send?(user, item) do - send( - socket.transport_pid, - {:text, represent_notification(socket.assigns[:user], item)} - ) - end - end) - - {:noreply, topics} - end - - def handle_cast(%{action: :stream, topic: "user", item: item}, topics) do - Logger.debug("Trying to push to users") - - recipient_topics = - User.get_recipients_from_activity(item) - |> Enum.map(fn %{id: id} -> "user:#{id}" end) - - Enum.each(recipient_topics, fn topic -> - push_to_socket(topics, topic, item) - end) - - {:noreply, topics} - end - - def handle_cast(%{action: :stream, topic: topic, item: item}, topics) do - Logger.debug("Trying to push to #{topic}") - Logger.debug("Pushing item to #{topic}") - push_to_socket(topics, topic, item) - {:noreply, topics} - end - - def handle_cast(%{action: :add, topic: topic, socket: socket}, sockets) do - topic = internal_topic(topic, socket) - sockets_for_topic = sockets[topic] || [] - sockets_for_topic = Enum.uniq([socket | sockets_for_topic]) - sockets = Map.put(sockets, topic, sockets_for_topic) - Logger.debug("Got new conn for #{topic}") - {:noreply, sockets} - end - - def handle_cast(%{action: :remove, topic: topic, socket: socket}, sockets) do - topic = internal_topic(topic, socket) - sockets_for_topic = sockets[topic] || [] - sockets_for_topic = List.delete(sockets_for_topic, socket) - sockets = Map.put(sockets, topic, sockets_for_topic) - Logger.debug("Removed conn for #{topic}") - {:noreply, sockets} - end - - def handle_cast(m, state) do - Logger.info("Unknown: #{inspect(m)}, #{inspect(state)}") - {:noreply, state} - end - - defp represent_update(%Activity{} = activity, %User{} = user) do - %{ - event: "update", - payload: - Pleroma.Web.MastodonAPI.StatusView.render( - "status.json", - activity: activity, - for: user - ) - |> Jason.encode!() - } - |> Jason.encode!() - end - - defp represent_update(%Activity{} = activity) do - %{ - event: "update", - payload: - Pleroma.Web.MastodonAPI.StatusView.render( - "status.json", - activity: activity - ) - |> Jason.encode!() - } - |> Jason.encode!() - end - - def represent_conversation(%Participation{} = participation) do - %{ - event: "conversation", - payload: - Pleroma.Web.MastodonAPI.ConversationView.render("participation.json", %{ - participation: participation, - for: participation.user - }) - |> Jason.encode!() - } - |> Jason.encode!() - end - - @spec represent_notification(User.t(), Notification.t()) :: binary() - defp represent_notification(%User{} = user, %Notification{} = notify) do - %{ - event: "notification", - payload: - NotificationView.render( - "show.json", - %{notification: notify, for: user} - ) - |> Jason.encode!() - } - |> Jason.encode!() - end - - defp should_send?(%User{} = user, %Activity{} = item) do - blocks = user.info.blocks || [] - mutes = user.info.mutes || [] - reblog_mutes = user.info.muted_reblogs || [] - domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(user.info.domain_blocks) - - with parent when not is_nil(parent) <- Object.normalize(item), - true <- Enum.all?([blocks, mutes, reblog_mutes], &(item.actor not in &1)), - true <- Enum.all?([blocks, mutes], &(parent.data["actor"] not in &1)), - %{host: item_host} <- URI.parse(item.actor), - %{host: parent_host} <- URI.parse(parent.data["actor"]), - false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host), - false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, parent_host), - true <- thread_containment(item, user), - false <- CommonAPI.thread_muted?(user, item) do - true - else - _ -> false - end - end - - defp should_send?(%User{} = user, %Notification{activity: activity}) do - should_send?(user, activity) - end - - def push_to_socket(topics, topic, %Activity{data: %{"type" => "Announce"}} = item) do - Enum.each(topics[topic] || [], fn socket -> - # Get the current user so we have up-to-date blocks etc. - if socket.assigns[:user] do - user = User.get_cached_by_ap_id(socket.assigns[:user].ap_id) - - if should_send?(user, item) do - send(socket.transport_pid, {:text, represent_update(item, user)}) - end - else - send(socket.transport_pid, {:text, represent_update(item)}) - end - end) - end - - def push_to_socket(topics, topic, %Participation{} = participation) do - Enum.each(topics[topic] || [], fn socket -> - send(socket.transport_pid, {:text, represent_conversation(participation)}) - end) - end - - def push_to_socket(topics, topic, %Activity{ - data: %{"type" => "Delete", "deleted_activity_id" => deleted_activity_id} - }) do - Enum.each(topics[topic] || [], fn socket -> - send( - socket.transport_pid, - {:text, %{event: "delete", payload: to_string(deleted_activity_id)} |> Jason.encode!()} - ) - end) - end - - def push_to_socket(_topics, _topic, %Activity{data: %{"type" => "Delete"}}), do: :noop - - def push_to_socket(topics, topic, item) do - Enum.each(topics[topic] || [], fn socket -> - # Get the current user so we have up-to-date blocks etc. - if socket.assigns[:user] do - user = User.get_cached_by_ap_id(socket.assigns[:user].ap_id) - blocks = user.info.blocks || [] - mutes = user.info.mutes || [] - - with true <- Enum.all?([blocks, mutes], &(item.actor not in &1)), - true <- thread_containment(item, user) do - send(socket.transport_pid, {:text, represent_update(item, user)}) - end - else - send(socket.transport_pid, {:text, represent_update(item)}) - end - end) - end - - defp internal_topic(topic, socket) when topic in ~w[user user:notification direct] do - "#{topic}:#{socket.assigns[:user].id}" - end - - defp internal_topic(topic, _), do: topic - - @spec thread_containment(Activity.t(), User.t()) :: boolean() - defp thread_containment(_activity, %User{info: %{skip_thread_containment: true}}), do: true - - defp thread_containment(activity, user) do - if Config.get([:instance, :skip_thread_containment]) do - true - else - ActivityPub.contain_activity(activity, user) - end - end -end diff --git a/lib/pleroma/web/streamer/ping.ex b/lib/pleroma/web/streamer/ping.ex new file mode 100644 index 000000000..f77cbb95c --- /dev/null +++ b/lib/pleroma/web/streamer/ping.ex @@ -0,0 +1,33 @@ +defmodule Pleroma.Web.Streamer.Ping do + use GenServer + require Logger + + alias Pleroma.Web.Streamer.State + alias Pleroma.Web.Streamer.StreamerSocket + + @keepalive_interval :timer.seconds(30) + + def start_link(opts) do + ping_interval = Keyword.get(opts, :ping_interval, @keepalive_interval) + GenServer.start_link(__MODULE__, %{ping_interval: ping_interval}, name: __MODULE__) + end + + def init(%{ping_interval: ping_interval} = args) do + Process.send_after(self(), :ping, ping_interval) + {:ok, args} + end + + def handle_info(:ping, %{ping_interval: ping_interval} = state) do + State.get_sockets() + |> Map.values() + |> List.flatten() + |> Enum.each(fn %StreamerSocket{transport_pid: transport_pid} -> + Logger.debug("Sending keepalive ping") + send(transport_pid, {:text, ""}) + end) + + Process.send_after(self(), :ping, ping_interval) + + {:noreply, state} + end +end diff --git a/lib/pleroma/web/streamer/state.ex b/lib/pleroma/web/streamer/state.ex new file mode 100644 index 000000000..7b5199068 --- /dev/null +++ b/lib/pleroma/web/streamer/state.ex @@ -0,0 +1,68 @@ +defmodule Pleroma.Web.Streamer.State do + use GenServer + require Logger + + alias Pleroma.Web.Streamer.StreamerSocket + + def start_link(_) do + GenServer.start_link(__MODULE__, %{sockets: %{}}, name: __MODULE__) + end + + def add_socket(topic, socket) do + GenServer.call(__MODULE__, {:add, socket, topic}) + end + + def remove_socket(topic, socket) do + GenServer.call(__MODULE__, {:remove, socket, topic}) + end + + def get_sockets do + %{sockets: stream_sockets} = GenServer.call(__MODULE__, :get_state) + stream_sockets + end + + def init(init_arg) do + {:ok, init_arg} + end + + def handle_call(:get_state, _from, state) do + {:reply, state, state} + end + + def handle_call({:add, socket, topic}, _from, %{sockets: sockets} = state) do + internal_topic = internal_topic(topic, socket) + stream_socket = StreamerSocket.from_socket(socket) + + sockets_for_topic = + sockets + |> Map.get(internal_topic, []) + |> List.insert_at(0, stream_socket) + |> Enum.uniq() + + state = put_in(state, [:sockets, internal_topic], sockets_for_topic) + Logger.debug("Got new conn for #{topic}") + {:reply, state, state} + end + + def handle_call({:remove, socket, topic}, _from, %{sockets: sockets} = state) do + internal_topic = internal_topic(topic, socket) + stream_socket = StreamerSocket.from_socket(socket) + + sockets_for_topic = + sockets + |> Map.get(internal_topic, []) + |> List.delete(stream_socket) + + state = Kernel.put_in(state, [:sockets, internal_topic], sockets_for_topic) + {:reply, state, state} + end + + defp internal_topic(topic, socket) + when topic in ~w[user user:notification direct] do + "#{topic}:#{socket.assigns[:user].id}" + end + + defp internal_topic(topic, _) do + topic + end +end diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex new file mode 100644 index 000000000..8cf719277 --- /dev/null +++ b/lib/pleroma/web/streamer/streamer.ex @@ -0,0 +1,55 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Streamer do + alias Pleroma.Web.Streamer.State + alias Pleroma.Web.Streamer.Worker + + @timeout 60_000 + @mix_env Mix.env() + + def add_socket(topic, socket) do + State.add_socket(topic, socket) + end + + def remove_socket(topic, socket) do + State.remove_socket(topic, socket) + end + + def get_sockets do + State.get_sockets() + end + + def stream(topics, items) do + if should_send?() do + Task.async(fn -> + :poolboy.transaction( + :streamer_worker, + &Worker.stream(&1, topics, items), + @timeout + ) + end) + end + end + + def supervisor, do: Pleroma.Web.Streamer.Supervisor + + defp should_send? do + handle_should_send(@mix_env) + end + + defp handle_should_send(:test) do + case Process.whereis(:streamer_worker) do + nil -> + false + + pid -> + Process.alive?(pid) + end + end + + defp handle_should_send(_) do + true + end +end diff --git a/lib/pleroma/web/streamer/streamer_socket.ex b/lib/pleroma/web/streamer/streamer_socket.ex new file mode 100644 index 000000000..f006c0306 --- /dev/null +++ b/lib/pleroma/web/streamer/streamer_socket.ex @@ -0,0 +1,31 @@ +defmodule Pleroma.Web.Streamer.StreamerSocket do + defstruct transport_pid: nil, user: nil + + alias Pleroma.User + alias Pleroma.Web.Streamer.StreamerSocket + + def from_socket(%{ + transport_pid: transport_pid, + assigns: %{user: nil} + }) do + %StreamerSocket{ + transport_pid: transport_pid + } + end + + def from_socket(%{ + transport_pid: transport_pid, + assigns: %{user: %User{} = user} + }) do + %StreamerSocket{ + transport_pid: transport_pid, + user: user + } + end + + def from_socket(%{transport_pid: transport_pid}) do + %StreamerSocket{ + transport_pid: transport_pid + } + end +end diff --git a/lib/pleroma/web/streamer/supervisor.ex b/lib/pleroma/web/streamer/supervisor.ex new file mode 100644 index 000000000..6afe19323 --- /dev/null +++ b/lib/pleroma/web/streamer/supervisor.ex @@ -0,0 +1,33 @@ +defmodule Pleroma.Web.Streamer.Supervisor do + use Supervisor + + def start_link(opts) do + Supervisor.start_link(__MODULE__, opts, name: __MODULE__) + end + + def init(args) do + children = [ + {Pleroma.Web.Streamer.State, args}, + {Pleroma.Web.Streamer.Ping, args}, + :poolboy.child_spec(:streamer_worker, poolboy_config()) + ] + + opts = [strategy: :one_for_one, name: Pleroma.Web.Streamer.Supervisor] + Supervisor.init(children, opts) + end + + defp poolboy_config do + opts = + Pleroma.Config.get(:streamer, + workers: 3, + overflow_workers: 2 + ) + + [ + {:name, {:local, :streamer_worker}}, + {:worker_module, Pleroma.Web.Streamer.Worker}, + {:size, opts[:workers]}, + {:max_overflow, opts[:overflow_workers]} + ] + end +end diff --git a/lib/pleroma/web/streamer/worker.ex b/lib/pleroma/web/streamer/worker.ex new file mode 100644 index 000000000..5804508eb --- /dev/null +++ b/lib/pleroma/web/streamer/worker.ex @@ -0,0 +1,220 @@ +defmodule Pleroma.Web.Streamer.Worker do + use GenServer + + require Logger + + alias Pleroma.Activity + alias Pleroma.Config + alias Pleroma.Conversation.Participation + alias Pleroma.Notification + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.Streamer.State + alias Pleroma.Web.Streamer.StreamerSocket + alias Pleroma.Web.StreamerView + + def start_link(_) do + GenServer.start_link(__MODULE__, %{}, []) + end + + def init(init_arg) do + {:ok, init_arg} + end + + def stream(pid, topics, items) do + GenServer.call(pid, {:stream, topics, items}) + end + + def handle_call({:stream, topics, item}, _from, state) when is_list(topics) do + Enum.each(topics, fn t -> + do_stream(%{topic: t, item: item}) + end) + + {:reply, state, state} + end + + def handle_call({:stream, topic, items}, _from, state) when is_list(items) do + Enum.each(items, fn i -> + do_stream(%{topic: topic, item: i}) + end) + + {:reply, state, state} + end + + def handle_call({:stream, topic, item}, _from, state) do + do_stream(%{topic: topic, item: item}) + + {:reply, state, state} + end + + defp do_stream(%{topic: "direct", item: item}) do + recipient_topics = + User.get_recipients_from_activity(item) + |> Enum.map(fn %{id: id} -> "direct:#{id}" end) + + Enum.each(recipient_topics, fn user_topic -> + Logger.debug("Trying to push direct message to #{user_topic}\n\n") + push_to_socket(State.get_sockets(), user_topic, item) + end) + end + + defp do_stream(%{topic: "participation", item: participation}) do + user_topic = "direct:#{participation.user_id}" + Logger.debug("Trying to push a conversation participation to #{user_topic}\n\n") + + push_to_socket(State.get_sockets(), user_topic, participation) + end + + defp do_stream(%{topic: "list", item: item}) do + # filter the recipient list if the activity is not public, see #270. + recipient_lists = + case Visibility.is_public?(item) do + true -> + Pleroma.List.get_lists_from_activity(item) + + _ -> + Pleroma.List.get_lists_from_activity(item) + |> Enum.filter(fn list -> + owner = User.get_cached_by_id(list.user_id) + + Visibility.visible_for_user?(item, owner) + end) + end + + recipient_topics = + recipient_lists + |> Enum.map(fn %{id: id} -> "list:#{id}" end) + + Enum.each(recipient_topics, fn list_topic -> + Logger.debug("Trying to push message to #{list_topic}\n\n") + push_to_socket(State.get_sockets(), list_topic, item) + end) + end + + defp do_stream(%{topic: topic, item: %Notification{} = item}) + when topic in ["user", "user:notification"] do + State.get_sockets() + |> Map.get("#{topic}:#{item.user_id}", []) + |> Enum.each(fn %StreamerSocket{transport_pid: transport_pid, user: socket_user} -> + with %User{} = user <- User.get_cached_by_ap_id(socket_user.ap_id), + true <- should_send?(user, item) do + send(transport_pid, {:text, StreamerView.render("notification.json", socket_user, item)}) + end + end) + end + + defp do_stream(%{topic: "user", item: item}) do + Logger.debug("Trying to push to users") + + recipient_topics = + User.get_recipients_from_activity(item) + |> Enum.map(fn %{id: id} -> "user:#{id}" end) + + Enum.each(recipient_topics, fn topic -> + push_to_socket(State.get_sockets(), topic, item) + end) + end + + defp do_stream(%{topic: topic, item: item}) do + Logger.debug("Trying to push to #{topic}") + Logger.debug("Pushing item to #{topic}") + push_to_socket(State.get_sockets(), topic, item) + end + + defp should_send?(%User{} = user, %Activity{} = item) do + blocks = user.info.blocks || [] + mutes = user.info.mutes || [] + reblog_mutes = user.info.muted_reblogs || [] + domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(user.info.domain_blocks) + + with parent when not is_nil(parent) <- Object.normalize(item), + true <- Enum.all?([blocks, mutes, reblog_mutes], &(item.actor not in &1)), + true <- Enum.all?([blocks, mutes], &(parent.data["actor"] not in &1)), + %{host: item_host} <- URI.parse(item.actor), + %{host: parent_host} <- URI.parse(parent.data["actor"]), + false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host), + false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, parent_host), + true <- thread_containment(item, user), + false <- CommonAPI.thread_muted?(user, item) do + true + else + _ -> false + end + end + + defp should_send?(%User{} = user, %Notification{activity: activity}) do + should_send?(user, activity) + end + + def push_to_socket(topics, topic, %Activity{data: %{"type" => "Announce"}} = item) do + Enum.each(topics[topic] || [], fn %StreamerSocket{ + transport_pid: transport_pid, + user: socket_user + } -> + # Get the current user so we have up-to-date blocks etc. + if socket_user do + user = User.get_cached_by_ap_id(socket_user.ap_id) + + if should_send?(user, item) do + send(transport_pid, {:text, StreamerView.render("update.json", item, user)}) + end + else + send(transport_pid, {:text, StreamerView.render("update.json", item)}) + end + end) + end + + def push_to_socket(topics, topic, %Participation{} = participation) do + Enum.each(topics[topic] || [], fn %StreamerSocket{transport_pid: transport_pid} -> + send(transport_pid, {:text, StreamerView.render("conversation.json", participation)}) + end) + end + + def push_to_socket(topics, topic, %Activity{ + data: %{"type" => "Delete", "deleted_activity_id" => deleted_activity_id} + }) do + Enum.each(topics[topic] || [], fn %StreamerSocket{transport_pid: transport_pid} -> + send( + transport_pid, + {:text, %{event: "delete", payload: to_string(deleted_activity_id)} |> Jason.encode!()} + ) + end) + end + + def push_to_socket(_topics, _topic, %Activity{data: %{"type" => "Delete"}}), do: :noop + + def push_to_socket(topics, topic, item) do + Enum.each(topics[topic] || [], fn %StreamerSocket{ + transport_pid: transport_pid, + user: socket_user + } -> + # Get the current user so we have up-to-date blocks etc. + if socket_user do + user = User.get_cached_by_ap_id(socket_user.ap_id) + blocks = user.info.blocks || [] + mutes = user.info.mutes || [] + + with true <- Enum.all?([blocks, mutes], &(item.actor not in &1)), + true <- thread_containment(item, user) do + send(transport_pid, {:text, StreamerView.render("update.json", item, user)}) + end + else + send(transport_pid, {:text, StreamerView.render("update.json", item)}) + end + end) + end + + @spec thread_containment(Activity.t(), User.t()) :: boolean() + defp thread_containment(_activity, %User{info: %{skip_thread_containment: true}}), do: true + + defp thread_containment(activity, user) do + if Config.get([:instance, :skip_thread_containment]) do + true + else + ActivityPub.contain_activity(activity, user) + end + end +end diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex new file mode 100644 index 000000000..b13030fa0 --- /dev/null +++ b/lib/pleroma/web/views/streamer_view.ex @@ -0,0 +1,66 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.StreamerView do + use Pleroma.Web, :view + + alias Pleroma.Activity + alias Pleroma.Conversation.Participation + alias Pleroma.Notification + alias Pleroma.User + alias Pleroma.Web.MastodonAPI.NotificationView + + def render("update.json", %Activity{} = activity, %User{} = user) do + %{ + event: "update", + payload: + Pleroma.Web.MastodonAPI.StatusView.render( + "status.json", + activity: activity, + for: user + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + + def render("notification.json", %User{} = user, %Notification{} = notify) do + %{ + event: "notification", + payload: + NotificationView.render( + "show.json", + %{notification: notify, for: user} + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + + def render("update.json", %Activity{} = activity) do + %{ + event: "update", + payload: + Pleroma.Web.MastodonAPI.StatusView.render( + "status.json", + activity: activity + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + + def render("conversation.json", %Participation{} = participation) do + %{ + event: "conversation", + payload: + Pleroma.Web.MastodonAPI.ConversationView.render("participation.json", %{ + participation: participation, + for: participation.user + }) + |> Jason.encode!() + } + |> Jason.encode!() + end +end diff --git a/mix.exs b/mix.exs index f1e98585b..911ebad1d 100644 --- a/mix.exs +++ b/mix.exs @@ -144,6 +144,7 @@ defmodule Pleroma.Mixfile do git: "https://git.pleroma.social/pleroma/http_signatures.git", ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"}, {:telemetry, "~> 0.3"}, + {:poolboy, "~> 1.5"}, {:prometheus_ex, "~> 3.0"}, {:prometheus_plugs, "~> 1.1"}, {:prometheus_phoenix, "~> 1.3"}, diff --git a/mix.lock b/mix.lock index 41697dd5c..0bf6a811e 100644 --- a/mix.lock +++ b/mix.lock @@ -73,6 +73,7 @@ "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"}, "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, + "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "prometheus": {:hex, :prometheus, "4.4.1", "1e96073b3ed7788053768fea779cbc896ddc3bdd9ba60687f2ad50b252ac87d6", [:mix, :rebar3], [], "hexpm"}, "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/activity/ir/topics_test.exs b/test/activity/ir/topics_test.exs new file mode 100644 index 000000000..e75f83586 --- /dev/null +++ b/test/activity/ir/topics_test.exs @@ -0,0 +1,141 @@ +defmodule Pleroma.Activity.Ir.TopicsTest do + use Pleroma.DataCase + + alias Pleroma.Activity + alias Pleroma.Activity.Ir.Topics + alias Pleroma.Object + + require Pleroma.Constants + + describe "poll answer" do + test "produce no topics" do + activity = %Activity{object: %Object{data: %{"type" => "Answer"}}} + + assert [] == Topics.get_activity_topics(activity) + end + end + + describe "non poll answer" do + test "always add user and list topics" do + activity = %Activity{object: %Object{data: %{"type" => "FooBar"}}} + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "user") + assert Enum.member?(topics, "list") + end + end + + describe "public visibility" do + setup do + activity = %Activity{ + object: %Object{data: %{"type" => "Note"}}, + data: %{"to" => [Pleroma.Constants.as_public()]} + } + + {:ok, activity: activity} + end + + test "produces public topic", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public") + end + + test "local action produces public:local topic", %{activity: activity} do + activity = %{activity | local: true} + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public:local") + end + + test "non-local action does not produce public:local topic", %{activity: activity} do + activity = %{activity | local: false} + topics = Topics.get_activity_topics(activity) + + refute Enum.member?(topics, "public:local") + end + end + + describe "public visibility create events" do + setup do + activity = %Activity{ + object: %Object{data: %{"type" => "Create", "attachment" => []}}, + data: %{"to" => [Pleroma.Constants.as_public()]} + } + + {:ok, activity: activity} + end + + test "with no attachments doesn't produce public:media topics", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + refute Enum.member?(topics, "public:media") + refute Enum.member?(topics, "public:local:media") + end + + test "converts tags to hash tags", %{activity: %{object: %{data: data} = object} = activity} do + tagged_data = Map.put(data, "tag", ["foo", "bar"]) + activity = %{activity | object: %{object | data: tagged_data}} + + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "hashtag:foo") + assert Enum.member?(topics, "hashtag:bar") + end + + test "only converts strinngs to hash tags", %{ + activity: %{object: %{data: data} = object} = activity + } do + tagged_data = Map.put(data, "tag", [2]) + activity = %{activity | object: %{object | data: tagged_data}} + + topics = Topics.get_activity_topics(activity) + + refute Enum.member?(topics, "hashtag:2") + end + end + + describe "public visibility create events with attachments" do + setup do + activity = %Activity{ + object: %Object{data: %{"type" => "Create", "attachment" => ["foo"]}}, + data: %{"to" => [Pleroma.Constants.as_public()]} + } + + {:ok, activity: activity} + end + + test "produce public:media topics", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public:media") + end + + test "local produces public:local:media topics", %{activity: activity} do + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "public:local:media") + end + + test "non-local doesn't produce public:local:media topics", %{activity: activity} do + activity = %{activity | local: false} + + topics = Topics.get_activity_topics(activity) + + refute Enum.member?(topics, "public:local:media") + end + end + + describe "non-public visibility" do + test "produces direct topic" do + activity = %Activity{object: %Object{data: %{"type" => "Note"}}, data: %{"to" => []}} + topics = Topics.get_activity_topics(activity) + + assert Enum.member?(topics, "direct") + refute Enum.member?(topics, "public") + refute Enum.member?(topics, "public:local") + refute Enum.member?(topics, "public:media") + refute Enum.member?(topics, "public:local:media") + end + end +end diff --git a/test/integration/mastodon_websocket_test.exs b/test/integration/mastodon_websocket_test.exs index 63bf73412..c04262808 100644 --- a/test/integration/mastodon_websocket_test.exs +++ b/test/integration/mastodon_websocket_test.exs @@ -11,7 +11,6 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do alias Pleroma.Integration.WebsocketClient alias Pleroma.Web.CommonAPI alias Pleroma.Web.OAuth - alias Pleroma.Web.Streamer @path Pleroma.Web.Endpoint.url() |> URI.parse() @@ -19,16 +18,6 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do |> Map.put(:path, "/api/v1/streaming") |> URI.to_string() - setup do - GenServer.start(Streamer, %{}, name: Streamer) - - on_exit(fn -> - if pid = Process.whereis(Streamer) do - Process.exit(pid, :kill) - end - end) - end - def start_socket(qs \\ nil, headers \\ []) do path = case qs do @@ -53,12 +42,14 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) end + @tag needs_streamer: true test "allows public streams without authentication" do assert {:ok, _} = start_socket("?stream=public") assert {:ok, _} = start_socket("?stream=public:local") assert {:ok, _} = start_socket("?stream=hashtag&tag=lain") end + @tag needs_streamer: true test "receives well formatted events" do user = insert(:user) {:ok, _} = start_socket("?stream=public") @@ -103,6 +94,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert {:ok, _} = start_socket("?stream=user&access_token=#{state.token.token}") end + @tag needs_streamer: true test "accepts the 'user' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user&access_token=#{token.token}") @@ -111,6 +103,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) =~ ":badarg" end + @tag needs_streamer: true test "accepts the 'user:notification' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user:notification&access_token=#{token.token}") @@ -119,6 +112,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) =~ ":badarg" end + @tag needs_streamer: true test "accepts valid token on Sec-WebSocket-Protocol header", %{token: token} do assert {:ok, _} = start_socket("?stream=user", [{"Sec-WebSocket-Protocol", token.token}]) diff --git a/test/notification_test.exs b/test/notification_test.exs index 3be9db09b..3d2f9a8fc 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -69,16 +69,7 @@ defmodule Pleroma.NotificationTest do end describe "create_notification" do - setup do - GenServer.start(Streamer, %{}, name: Streamer) - - on_exit(fn -> - if pid = Process.whereis(Streamer) do - Process.exit(pid, :kill) - end - end) - end - + @tag needs_streamer: true test "it creates a notification for user and send to the 'user' and the 'user:notification' stream" do user = insert(:user) task = Task.async(fn -> assert_receive {:text, _}, 4_000 end) diff --git a/test/support/conn_case.ex b/test/support/conn_case.ex index ec5892ff5..b39c70677 100644 --- a/test/support/conn_case.ex +++ b/test/support/conn_case.ex @@ -40,6 +40,10 @@ defmodule Pleroma.Web.ConnCase do Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, {:shared, self()}) end + if tags[:needs_streamer] do + start_supervised(Pleroma.Web.Streamer.supervisor()) + end + {:ok, conn: Phoenix.ConnTest.build_conn()} end end diff --git a/test/support/data_case.ex b/test/support/data_case.ex index f3d98e7e3..17fa15214 100644 --- a/test/support/data_case.ex +++ b/test/support/data_case.ex @@ -39,6 +39,10 @@ defmodule Pleroma.DataCase do Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, {:shared, self()}) end + if tags[:needs_streamer] do + start_supervised(Pleroma.Web.Streamer.supervisor()) + end + :ok end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index d0118fefa..4100108a5 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -38,9 +38,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do stream: fn _, _ -> nil end do ActivityPub.stream_out_participations(conversation.participations) - Enum.each(participations, fn participation -> - assert called(Pleroma.Web.Streamer.stream("participation", participation)) - end) + assert called(Pleroma.Web.Streamer.stream("participation", participations)) end end end diff --git a/test/web/streamer/ping_test.exs b/test/web/streamer/ping_test.exs new file mode 100644 index 000000000..3d52c00e4 --- /dev/null +++ b/test/web/streamer/ping_test.exs @@ -0,0 +1,36 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PingTest do + use Pleroma.DataCase + + import Pleroma.Factory + alias Pleroma.Web.Streamer + + setup do + start_supervised({Streamer.supervisor(), [ping_interval: 30]}) + + :ok + end + + describe "sockets" do + setup do + user = insert(:user) + {:ok, %{user: user}} + end + + test "it sends pings", %{user: user} do + task = + Task.async(fn -> + assert_receive {:text, received_event}, 40 + assert_receive {:text, received_event}, 40 + assert_receive {:text, received_event}, 40 + end) + + Streamer.add_socket("public", %{transport_pid: task.pid, assigns: %{user: user}}) + + Task.await(task) + end + end +end diff --git a/test/web/streamer/state_test.exs b/test/web/streamer/state_test.exs new file mode 100644 index 000000000..d1aeac541 --- /dev/null +++ b/test/web/streamer/state_test.exs @@ -0,0 +1,54 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.StateTest do + use Pleroma.DataCase + + import Pleroma.Factory + alias Pleroma.Web.Streamer + alias Pleroma.Web.Streamer.StreamerSocket + + @moduletag needs_streamer: true + + describe "sockets" do + setup do + user = insert(:user) + user2 = insert(:user) + {:ok, %{user: user, user2: user2}} + end + + test "it can add a socket", %{user: user} do + Streamer.add_socket("public", %{transport_pid: 1, assigns: %{user: user}}) + + assert(%{"public" => [%StreamerSocket{transport_pid: 1}]} = Streamer.get_sockets()) + end + + test "it can add multiple sockets per user", %{user: user} do + Streamer.add_socket("public", %{transport_pid: 1, assigns: %{user: user}}) + Streamer.add_socket("public", %{transport_pid: 2, assigns: %{user: user}}) + + assert( + %{ + "public" => [ + %StreamerSocket{transport_pid: 2}, + %StreamerSocket{transport_pid: 1} + ] + } = Streamer.get_sockets() + ) + end + + test "it will not add a duplicate socket", %{user: user} do + Streamer.add_socket("activity", %{transport_pid: 1, assigns: %{user: user}}) + Streamer.add_socket("activity", %{transport_pid: 1, assigns: %{user: user}}) + + assert( + %{ + "activity" => [ + %StreamerSocket{transport_pid: 1} + ] + } = Streamer.get_sockets() + ) + end + end +end diff --git a/test/web/streamer_test.exs b/test/web/streamer/streamer_test.exs similarity index 86% rename from test/web/streamer_test.exs rename to test/web/streamer/streamer_test.exs index 96fa7645f..88847e20f 100644 --- a/test/web/streamer_test.exs +++ b/test/web/streamer/streamer_test.exs @@ -5,24 +5,20 @@ defmodule Pleroma.Web.StreamerTest do use Pleroma.DataCase + import Pleroma.Factory + alias Pleroma.List alias Pleroma.User alias Pleroma.Web.CommonAPI alias Pleroma.Web.Streamer - import Pleroma.Factory + alias Pleroma.Web.Streamer.StreamerSocket + alias Pleroma.Web.Streamer.Worker + @moduletag needs_streamer: true clear_config_all([:instance, :skip_thread_containment]) describe "user streams" do setup do - GenServer.start(Streamer, %{}, name: Streamer) - - on_exit(fn -> - if pid = Process.whereis(Streamer) do - Process.exit(pid, :kill) - end - end) - user = insert(:user) notify = insert(:notification, user: user, activity: build(:note_activity)) {:ok, %{user: user, notify: notify}} @@ -125,11 +121,9 @@ defmodule Pleroma.Web.StreamerTest do assert_receive {:text, _}, 4_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user - } + user: user } {:ok, activity} = CommonAPI.post(other_user, %{"status" => "Test"}) @@ -138,7 +132,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) @@ -155,11 +149,9 @@ defmodule Pleroma.Web.StreamerTest do assert received_event == expected_event end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user - } + user: user } {:ok, activity} = CommonAPI.delete(activity.id, other_user) @@ -168,7 +160,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) end @@ -189,9 +181,9 @@ defmodule Pleroma.Web.StreamerTest do ) task = Task.async(fn -> refute_receive {:text, _}, 1_000 end) - fake_socket = %{transport_pid: task.pid, assigns: %{user: user}} + fake_socket = %StreamerSocket{transport_pid: task.pid, user: user} topics = %{"public" => [fake_socket]} - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) end @@ -211,9 +203,9 @@ defmodule Pleroma.Web.StreamerTest do ) task = Task.async(fn -> assert_receive {:text, _}, 1_000 end) - fake_socket = %{transport_pid: task.pid, assigns: %{user: user}} + fake_socket = %StreamerSocket{transport_pid: task.pid, user: user} topics = %{"public" => [fake_socket]} - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) end @@ -233,9 +225,9 @@ defmodule Pleroma.Web.StreamerTest do ) task = Task.async(fn -> assert_receive {:text, _}, 1_000 end) - fake_socket = %{transport_pid: task.pid, assigns: %{user: user}} + fake_socket = %StreamerSocket{transport_pid: task.pid, user: user} topics = %{"public" => [fake_socket]} - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) end @@ -251,11 +243,9 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user - } + user: user } {:ok, activity} = CommonAPI.post(blocked_user, %{"status" => "Test"}) @@ -264,7 +254,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Streamer.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) Task.await(task) end @@ -284,11 +274,9 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user_a - } + user: user_a } {:ok, activity} = @@ -301,7 +289,7 @@ defmodule Pleroma.Web.StreamerTest do "list:#{list.id}" => [fake_socket] } - Streamer.handle_cast(%{action: :stream, topic: "list", item: activity}, topics) + Worker.handle_call({:stream, "list", activity}, self(), topics) Task.await(task) end @@ -318,11 +306,9 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user_a - } + user: user_a } {:ok, activity} = @@ -335,12 +321,12 @@ defmodule Pleroma.Web.StreamerTest do "list:#{list.id}" => [fake_socket] } - Streamer.handle_cast(%{action: :stream, topic: "list", item: activity}, topics) + Worker.handle_call({:stream, "list", activity}, self(), topics) Task.await(task) end - test "it send wanted private posts to list" do + test "it sends wanted private posts to list" do user_a = insert(:user) user_b = insert(:user) @@ -354,11 +340,9 @@ defmodule Pleroma.Web.StreamerTest do assert_receive {:text, _}, 1_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user_a - } + user: user_a } {:ok, activity} = @@ -367,11 +351,12 @@ defmodule Pleroma.Web.StreamerTest do "visibility" => "private" }) - topics = %{ - "list:#{list.id}" => [fake_socket] - } + Streamer.add_socket( + "list:#{list.id}", + fake_socket + ) - Streamer.handle_cast(%{action: :stream, topic: "list", item: activity}, topics) + Worker.handle_call({:stream, "list", activity}, self(), %{}) Task.await(task) end @@ -387,11 +372,9 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 1_000 end) - fake_socket = %{ + fake_socket = %StreamerSocket{ transport_pid: task.pid, - assigns: %{ - user: user1 - } + user: user1 } {:ok, create_activity} = CommonAPI.post(user3, %{"status" => "I'm kawen"}) @@ -401,7 +384,7 @@ defmodule Pleroma.Web.StreamerTest do "public" => [fake_socket] } - Streamer.push_to_socket(topics, "public", announce_activity) + Worker.push_to_socket(topics, "public", announce_activity) Task.await(task) end @@ -417,6 +400,8 @@ defmodule Pleroma.Web.StreamerTest do task = Task.async(fn -> refute_receive {:text, _}, 4_000 end) + Process.sleep(4000) + Streamer.add_socket( "user", %{transport_pid: task.pid, assigns: %{user: user2}} @@ -428,14 +413,6 @@ defmodule Pleroma.Web.StreamerTest do describe "direct streams" do setup do - GenServer.start(Streamer, %{}, name: Streamer) - - on_exit(fn -> - if pid = Process.whereis(Streamer) do - Process.exit(pid, :kill) - end - end) - :ok end @@ -480,6 +457,8 @@ defmodule Pleroma.Web.StreamerTest do refute_receive {:text, _}, 4_000 end) + Process.sleep(1000) + Streamer.add_socket( "direct", %{transport_pid: task.pid, assigns: %{user: user}} @@ -521,6 +500,8 @@ defmodule Pleroma.Web.StreamerTest do assert last_status["id"] == to_string(create_activity.id) end) + Process.sleep(1000) + Streamer.add_socket( "direct", %{transport_pid: task.pid, assigns: %{user: user}} From 085d014f0859b3b3e5023c423ae0361ec6ed6c67 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 16 Sep 2019 19:26:00 +0700 Subject: [PATCH 40/45] Fix `Transmogrifier.upgrade_user_from_ap_id/1` --- lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index acb3087d0..8461b666e 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -1050,7 +1050,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do with %User{local: false} = user <- User.get_cached_by_ap_id(ap_id), {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id), already_ap <- User.ap_enabled?(user), - {:ok, user} <- user |> User.upgrade_changeset(data) |> User.update_and_set_cache() do + {:ok, user} <- user |> User.upgrade_changeset(data, true) |> User.update_and_set_cache() do unless already_ap do TransmogrifierWorker.enqueue("user_upgrade", %{"user_id" => user.id}) end From d6ab78e610f16e97246ec9e83b3db72f04cf41e7 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 16 Sep 2019 21:48:01 +0700 Subject: [PATCH 41/45] Set `account_field_value_length` limit to 2048 by default --- config/config.exs | 2 +- config/description.exs | 4 ++-- docs/config.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/config.exs b/config/config.exs index b1b98af93..c7e0cf09f 100644 --- a/config/config.exs +++ b/config/config.exs @@ -276,7 +276,7 @@ config :pleroma, :instance, max_account_fields: 10, max_remote_account_fields: 20, account_field_name_length: 512, - account_field_value_length: 512, + account_field_value_length: 2048, external_user_synchronization: true config :pleroma, :markup, diff --git a/config/description.exs b/config/description.exs index be5eb0cc3..32d36d6d6 100644 --- a/config/description.exs +++ b/config/description.exs @@ -878,9 +878,9 @@ config :pleroma, :config_description, [ %{ key: :account_field_value_length, type: :integer, - description: "An account field value maximum length (default: 512)", + description: "An account field value maximum length (default: 2048)", suggestions: [ - 512 + 2048 ] }, %{ diff --git a/docs/config.md b/docs/config.md index 270d7fcea..3f37fa561 100644 --- a/docs/config.md +++ b/docs/config.md @@ -135,7 +135,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `max_account_fields`: The maximum number of custom fields in the user profile (default: `10`) * `max_remote_account_fields`: The maximum number of custom fields in the remote user profile (default: `20`) * `account_field_name_length`: An account field name maximum length (default: `512`) -* `account_field_value_length`: An account field value maximum length (default: `512`) +* `account_field_value_length`: An account field value maximum length (default: `2048`) * `external_user_synchronization`: Enabling following/followers counters synchronization for external users. From a21584556f2c3edb90db3c58ba2a4829a7e220c1 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 17 Sep 2019 13:04:43 +0300 Subject: [PATCH 42/45] Update oban to 0.8.1 This version uses a different locking mechanism, which gets rid of `WARNING: you don't own a lock of type ShareLock` log spam --- lib/pleroma/flake_id.ex | 2 +- mix.exs | 2 +- mix.lock | 8 ++++---- priv/repo/migrations/20190917100019_update_oban.exs | 11 +++++++++++ 4 files changed, 17 insertions(+), 6 deletions(-) create mode 100644 priv/repo/migrations/20190917100019_update_oban.exs diff --git a/lib/pleroma/flake_id.ex b/lib/pleroma/flake_id.ex index 47d61ca5f..042cf8659 100644 --- a/lib/pleroma/flake_id.ex +++ b/lib/pleroma/flake_id.ex @@ -14,7 +14,7 @@ defmodule Pleroma.FlakeId do @type t :: binary - @behaviour Ecto.Type + use Ecto.Type use GenServer require Logger alias __MODULE__ diff --git a/mix.exs b/mix.exs index 911ebad1d..230f90244 100644 --- a/mix.exs +++ b/mix.exs @@ -101,7 +101,7 @@ defmodule Pleroma.Mixfile do {:phoenix_ecto, "~> 4.0"}, {:ecto_sql, "~> 3.1"}, {:postgrex, ">= 0.13.5"}, - {:oban, "~> 0.7"}, + {:oban, "~> 0.8.1"}, {:quantum, "~> 2.3"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, diff --git a/mix.lock b/mix.lock index 0bf6a811e..547ff6be6 100644 --- a/mix.lock +++ b/mix.lock @@ -21,8 +21,8 @@ "decimal": {:hex, :decimal, "1.8.0", "ca462e0d885f09a1c5a342dbd7c1dcf27ea63548c65a65e67334f4b61803822e", [:mix], [], "hexpm"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, "earmark": {:hex, :earmark, "1.3.6", "ce1d0675e10a5bb46b007549362bd3f5f08908843957687d8484fe7f37466b19", [:mix], [], "hexpm"}, - "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, - "ecto_sql": {:hex, :ecto_sql, "3.1.3", "2c536139190492d9de33c5fefac7323c5eaaa82e1b9bf93482a14649042f7cd9", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "ecto": {:hex, :ecto, "3.2.0", "940e2598813f205223d60c78d66e514afe1db5167ed8075510a59e496619cfb5", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "ecto_sql": {:hex, :ecto_sql, "3.2.0", "751cea597e8deb616084894dd75cbabfdbe7255ff01e8c058ca13f0353a3921b", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.2.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"}, "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"}, "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"}, @@ -60,7 +60,7 @@ "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm"}, "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm"}, "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"}, - "oban": {:hex, :oban, "0.7.1", "171bdd1b69c1a4a839f8c768f5e962fc22d1de1513d459fb6b8e0cbd34817a9a", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "oban": {:hex, :oban, "0.8.1", "4bbf62eb1829f856d69aeb5069ac7036afe07db8221a17de2a9169cc7a58a318", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.3", "6706a148809a29c306062862c803406e88f048277f6e85b68faf73291e820b84", [:mix], [], "hexpm"}, "phoenix": {:hex, :phoenix, "1.4.9", "746d098e10741c334d88143d3c94cab1756435f94387a63441792e66ec0ee974", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, @@ -74,7 +74,7 @@ "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, - "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "postgrex": {:hex, :postgrex, "0.15.1", "23ce3417de70f4c0e9e7419ad85bdabcc6860a6925fe2c6f3b1b5b1e8e47bf2f", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "prometheus": {:hex, :prometheus, "4.4.1", "1e96073b3ed7788053768fea779cbc896ddc3bdd9ba60687f2ad50b252ac87d6", [:mix, :rebar3], [], "hexpm"}, "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/priv/repo/migrations/20190917100019_update_oban.exs b/priv/repo/migrations/20190917100019_update_oban.exs new file mode 100644 index 000000000..157dc54f9 --- /dev/null +++ b/priv/repo/migrations/20190917100019_update_oban.exs @@ -0,0 +1,11 @@ +defmodule Pleroma.Repo.Migrations.UpdateOban do + use Ecto.Migration + + def up do + Oban.Migrations.up(version: 4) + end + + def down do + Oban.Migrations.down(version: 2) + end +end From 80c5c3495bdd7939e576c8746a959f3f89f44042 Mon Sep 17 00:00:00 2001 From: Steven Fuchs Date: Tue, 17 Sep 2019 14:44:52 +0000 Subject: [PATCH 43/45] remove remaining errors from tests --- lib/pleroma/application.ex | 53 ++++++++++++++------ lib/pleroma/web/streamer/state.ex | 18 +++++-- test/integration/mastodon_websocket_test.exs | 16 ++++-- 3 files changed, 62 insertions(+), 25 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 3b37ce630..dabce771d 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -43,23 +43,9 @@ defmodule Pleroma.Application do hackney_pool_children() ++ [ Pleroma.Stats, - {Oban, Pleroma.Config.get(Oban)}, - %{ - id: :web_push_init, - start: {Task, :start_link, [&Pleroma.Web.Push.init/0]}, - restart: :temporary - }, - %{ - id: :federator_init, - start: {Task, :start_link, [&Pleroma.Web.Federator.init/0]}, - restart: :temporary - }, - %{ - id: :internal_fetch_init, - start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]}, - restart: :temporary - } + {Oban, Pleroma.Config.get(Oban)} ] ++ + task_children(@env) ++ oauth_cleanup_child(oauth_cleanup_enabled?()) ++ streamer_child(@env) ++ chat_child(@env, chat_enabled?()) ++ @@ -163,4 +149,39 @@ defmodule Pleroma.Application do :hackney_pool.child_spec(pool, options) end end + + defp task_children(:test) do + [ + %{ + id: :web_push_init, + start: {Task, :start_link, [&Pleroma.Web.Push.init/0]}, + restart: :temporary + }, + %{ + id: :federator_init, + start: {Task, :start_link, [&Pleroma.Web.Federator.init/0]}, + restart: :temporary + } + ] + end + + defp task_children(_) do + [ + %{ + id: :web_push_init, + start: {Task, :start_link, [&Pleroma.Web.Push.init/0]}, + restart: :temporary + }, + %{ + id: :federator_init, + start: {Task, :start_link, [&Pleroma.Web.Federator.init/0]}, + restart: :temporary + }, + %{ + id: :internal_fetch_init, + start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]}, + restart: :temporary + } + ] + end end diff --git a/lib/pleroma/web/streamer/state.ex b/lib/pleroma/web/streamer/state.ex index 7b5199068..c48752d95 100644 --- a/lib/pleroma/web/streamer/state.ex +++ b/lib/pleroma/web/streamer/state.ex @@ -4,16 +4,18 @@ defmodule Pleroma.Web.Streamer.State do alias Pleroma.Web.Streamer.StreamerSocket + @env Mix.env() + def start_link(_) do GenServer.start_link(__MODULE__, %{sockets: %{}}, name: __MODULE__) end def add_socket(topic, socket) do - GenServer.call(__MODULE__, {:add, socket, topic}) + GenServer.call(__MODULE__, {:add, topic, socket}) end def remove_socket(topic, socket) do - GenServer.call(__MODULE__, {:remove, socket, topic}) + do_remove_socket(@env, topic, socket) end def get_sockets do @@ -29,7 +31,7 @@ defmodule Pleroma.Web.Streamer.State do {:reply, state, state} end - def handle_call({:add, socket, topic}, _from, %{sockets: sockets} = state) do + def handle_call({:add, topic, socket}, _from, %{sockets: sockets} = state) do internal_topic = internal_topic(topic, socket) stream_socket = StreamerSocket.from_socket(socket) @@ -44,7 +46,7 @@ defmodule Pleroma.Web.Streamer.State do {:reply, state, state} end - def handle_call({:remove, socket, topic}, _from, %{sockets: sockets} = state) do + def handle_call({:remove, topic, socket}, _from, %{sockets: sockets} = state) do internal_topic = internal_topic(topic, socket) stream_socket = StreamerSocket.from_socket(socket) @@ -57,6 +59,14 @@ defmodule Pleroma.Web.Streamer.State do {:reply, state, state} end + defp do_remove_socket(:test, _, _) do + :ok + end + + defp do_remove_socket(_env, topic, socket) do + GenServer.call(__MODULE__, {:remove, topic, socket}) + end + defp internal_topic(topic, socket) when topic in ~w[user user:notification direct] do "#{topic}:#{socket.assigns[:user].id}" diff --git a/test/integration/mastodon_websocket_test.exs b/test/integration/mastodon_websocket_test.exs index c04262808..d02a3cc4d 100644 --- a/test/integration/mastodon_websocket_test.exs +++ b/test/integration/mastodon_websocket_test.exs @@ -18,6 +18,11 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do |> Map.put(:path, "/api/v1/streaming") |> URI.to_string() + setup_all do + start_supervised(Pleroma.Web.Streamer.supervisor()) + :ok + end + def start_socket(qs \\ nil, headers \\ []) do path = case qs do @@ -32,6 +37,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do capture_log(fn -> assert {:error, {400, _}} = start_socket() assert {:error, {404, _}} = start_socket("?stream=ncjdk") + Process.sleep(30) end) end @@ -39,17 +45,16 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do capture_log(fn -> assert {:error, {403, _}} = start_socket("?stream=user&access_token=aaaaaaaaaaaa") assert {:error, {403, _}} = start_socket("?stream=user") + Process.sleep(30) end) end - @tag needs_streamer: true test "allows public streams without authentication" do assert {:ok, _} = start_socket("?stream=public") assert {:ok, _} = start_socket("?stream=public:local") assert {:ok, _} = start_socket("?stream=hashtag&tag=lain") end - @tag needs_streamer: true test "receives well formatted events" do user = insert(:user) {:ok, _} = start_socket("?stream=public") @@ -94,31 +99,32 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert {:ok, _} = start_socket("?stream=user&access_token=#{state.token.token}") end - @tag needs_streamer: true test "accepts the 'user' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user&access_token=#{token.token}") assert capture_log(fn -> assert {:error, {403, "Forbidden"}} = start_socket("?stream=user") + Process.sleep(30) end) =~ ":badarg" end - @tag needs_streamer: true test "accepts the 'user:notification' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user:notification&access_token=#{token.token}") assert capture_log(fn -> assert {:error, {403, "Forbidden"}} = start_socket("?stream=user:notification") + Process.sleep(30) end) =~ ":badarg" end - @tag needs_streamer: true test "accepts valid token on Sec-WebSocket-Protocol header", %{token: token} do assert {:ok, _} = start_socket("?stream=user", [{"Sec-WebSocket-Protocol", token.token}]) assert capture_log(fn -> assert {:error, {403, "Forbidden"}} = start_socket("?stream=user", [{"Sec-WebSocket-Protocol", "I am a friend"}]) + + Process.sleep(30) end) =~ ":badarg" end end From 6193157f1998b10ac6cb9f4d36dd863eced81b37 Mon Sep 17 00:00:00 2001 From: Steven Fuchs Date: Tue, 17 Sep 2019 18:12:27 +0000 Subject: [PATCH 44/45] Fix notification warnings --- lib/pleroma/workers/web_pusher_worker.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/workers/web_pusher_worker.ex b/lib/pleroma/workers/web_pusher_worker.ex index bea2baffb..61b451e3e 100644 --- a/lib/pleroma/workers/web_pusher_worker.ex +++ b/lib/pleroma/workers/web_pusher_worker.ex @@ -10,7 +10,11 @@ defmodule Pleroma.Workers.WebPusherWorker do @impl Oban.Worker def perform(%{"op" => "web_push", "notification_id" => notification_id}, _job) do - notification = Repo.get(Notification, notification_id) + notification = + Notification + |> Repo.get(notification_id) + |> Repo.preload([:activity]) + Pleroma.Web.Push.Impl.perform(notification) end end From d201eec45cc5eb8c7b0c912c14be4704dbb4c1b1 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 17 Sep 2019 22:02:37 +0300 Subject: [PATCH 45/45] fixed ecto version --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 230f90244..58d1606d3 100644 --- a/mix.exs +++ b/mix.exs @@ -99,7 +99,7 @@ defmodule Pleroma.Mixfile do {:plug_cowboy, "~> 2.0"}, {:phoenix_pubsub, "~> 1.1"}, {:phoenix_ecto, "~> 4.0"}, - {:ecto_sql, "~> 3.1"}, + {:ecto_sql, "~> 3.2"}, {:postgrex, ">= 0.13.5"}, {:oban, "~> 0.8.1"}, {:quantum, "~> 2.3"},