Merge branch 'develop' into 'remove-twitter-api'

# Conflicts:
#   lib/pleroma/web/nodeinfo/nodeinfo_controller.ex

.gitlab-ci.yml

@@ -48,6 +48,7 @@ benchmark:
 
 unit-testing:
   stage: test
+  retry: 2
   cache: &testing_cache_policy
     <<: *global_cache_policy
     policy: pull
@@ -80,6 +81,7 @@ unit-testing:
 
 unit-testing-rum:
   stage: test
+  retry: 2
   cache: *testing_cache_policy
   services:
   - name: minibikini/postgres-with-rum:12

CHANGELOG.md

@@ -4,23 +4,127 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [unreleased]
+
 ### Changed
-- **Breaking:** BBCode and Markdown formatters will no longer return any `\n` and only use `<br/>` for newlines
+<details>
+  <summary>API Changes</summary>
+- **Breaking:** Emoji API: changed methods and renamed routes.
+</details>
 
 ### Removed
 - **Breaking:** removed `with_move` parameter from notifications timeline.
 
 ### Added
+- Instance: Extend `/api/v1/instance` with Pleroma-specific information.
 - NodeInfo: `pleroma:api/v1/notifications:include_types_filter` to the `features` list.
+- NodeInfo: `pleroma_emoji_reactions` to the `features` list.
 - Configuration: `:restrict_unauthenticated` setting, restrict access for unauthenticated users to timelines (public and federate), user profiles and statuses.
+- Configuration: Add `:database_config_whitelist` setting to whitelist settings which can be configured from AdminFE.
+- New HTTP adapter [gun](https://github.com/ninenines/gun). Gun adapter requires minimum OTP version of 22.2 otherwise Pleroma won’t start. For hackney OTP update is not required.
+- Mix task to create trusted OAuth App.
+- Notifications: Added `follow_request` notification type.
+- Added `:reject_deletes` group to SimplePolicy
 <details>
   <summary>API Changes</summary>
+- Mastodon API: Extended `/api/v1/instance`.
 - Mastodon API: Support for `include_types` in `/api/v1/notifications`.
+- Mastodon API: Added `/api/v1/notifications/:id/dismiss` endpoint.
+- Mastodon API: Add support for filtering replies in public and home timelines
+- Admin API: endpoints for create/update/delete OAuth Apps.
+- Admin API: endpoint for status view.
 </details>
 
+### Fixed
+- Support pagination in conversations API
+- **Breaking**: SimplePolicy `:reject` and `:accept` allow deletions again
+- Fix follower/blocks import when nicknames starts with @
+- Filtering of push notifications on activities from blocked domains
+- Resolving Peertube accounts with Webfinger
+
+## [Unreleased (patch)]
+
+### Fixed
+- Healthcheck reporting the number of memory currently used, rather than allocated in total
+- `InsertSkeletonsForDeletedUsers` failing on some instances 
+
+## [2.0.3] - 2020-05-02
+
+### Security
+- Disallow re-registration of previously deleted users, which allowed viewing direct messages addressed to them
+- Mastodon API: Fix `POST /api/v1/follow_requests/:id/authorize` allowing to force a follow from a local user even if they didn't request to follow
+- CSP: Sandbox uploads
+
+### Fixed
+- Notifications from blocked domains
+- Potential federation issues with Mastodon versions before 3.0.0
+- HTTP Basic Authentication permissions issue
+- Follow/Block imports not being able to find the user if the nickname started with an `@`
+- Instance stats counting internal users
+- Inability to run a From Source release without git
+- ObjectAgePolicy didn't filter out old messages
+- `blob:` urls not being allowed by CSP
+
+### Added
+- NodeInfo: ObjectAgePolicy settings to the `federation` list.
+- Follow request notifications
+<details>
+  <summary>API Changes</summary>
+- Admin API: `GET /api/pleroma/admin/need_reboot`.
+</details>
+
+### Upgrade notes
+
+1. Restart Pleroma
+2. Run database migrations (inside Pleroma directory):
+  - OTP: `./bin/pleroma_ctl migrate`
+  - From Source: `mix ecto.migrate`
+
+
+## [2.0.2] - 2020-04-08
+### Added
+- Support for Funkwhale's `Audio` activity
+- Admin API: `PATCH /api/pleroma/admin/users/:nickname/update_credentials`
+
+### Fixed
+- Blocked/muted users still generating push notifications
+- Input textbox for bio ignoring newlines
+- OTP: Inability to use PostgreSQL databases with SSL
+- `user delete_activities` breaking when trying to delete already deleted posts
+- Incorrect URL for Funkwhale channels
+
+### Upgrade notes
+1. Restart Pleroma
+
+## [2.0.1] - 2020-03-15
+### Security
+- Static-FE: Fix remote posts not being sanitized
+
+### Fixed
+- 500 errors when no `Accept` header is present if Static-FE is enabled
+- Instance panel not being updated immediately due to wrong `Cache-Control` headers
+- Statuses posted with BBCode/Markdown having unncessary newlines in Pleroma-FE
+- OTP: Fix some settings not being migrated to in-database config properly
+- No `Cache-Control` headers on attachment/media proxy requests
+- Character limit enforcement being off by 1
+- Mastodon Streaming API: hashtag timelines not working
+
+### Changed
+- BBCode and Markdown formatters will no longer return any `\n` and only use `<br/>` for newlines
+- Mastodon API: Allow registration without email if email verification is not enabled
+
+### Upgrade notes
+#### Nginx only
+1. Remove `proxy_ignore_headers Cache-Control;` and `proxy_hide_header  Cache-Control;` from your config.
+
+#### Everyone
+1. Run database migrations (inside Pleroma directory):
+  - OTP: `./bin/pleroma_ctl migrate`
+  - From Source: `mix ecto.migrate`
+2. Restart Pleroma
+
 ## [2.0.0] - 2019-03-08
 ### Security
-- Mastodon API: Fix being able to request enourmous amount of statuses in timelines leading to DoS. Now limited to 40 per request.
+- Mastodon API: Fix being able to request enormous amount of statuses in timelines leading to DoS. Now limited to 40 per request.
 
 ### Removed
 - **Breaking**: Removed 1.0+ deprecated configurations `Pleroma.Upload, :strip_exif` and `:instance, :dedupe_media`
@@ -64,7 +168,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - **Breaking:** Admin API: Return link alongside with token on password reset
 - **Breaking:** Admin API: `PUT /api/pleroma/admin/reports/:id` is now `PATCH /api/pleroma/admin/reports`, see admin_api.md for details
 - **Breaking:** `/api/pleroma/admin/users/invite_token` now uses `POST`, changed accepted params and returns full invite in json instead of only token string.
-- **Breaking** replying to reports is now "report notes", enpoint changed from `POST /api/pleroma/admin/reports/:id/respond` to `POST /api/pleroma/admin/reports/:id/notes`
+- **Breaking** replying to reports is now "report notes", endpoint changed from `POST /api/pleroma/admin/reports/:id/respond` to `POST /api/pleroma/admin/reports/:id/notes`
 - Mastodon API: stopped sanitizing display names, field names and subject fields since they are supposed to be treated as plaintext
 - Admin API: Return `total` when querying for reports
 - Mastodon API: Return `pleroma.direct_conversation_id` when creating a direct message (`POST /api/v1/statuses`)
@@ -74,10 +178,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: `pleroma.thread_muted` to the Status entity
 - Mastodon API: Mark the direct conversation as read for the author when they send a new direct message
 - Mastodon API, streaming: Add `pleroma.direct_conversation_id` to the `conversation` stream event payload.
+- Mastodon API: Add `pleroma.unread_count` to the Marker entity
 - Admin API: Render whole status in grouped reports
 - Mastodon API: User timelines will now respect blocks, unless you are getting the user timeline of somebody you blocked (which would be empty otherwise).
 - Mastodon API: Favoriting / Repeating a post multiple times will now return the identical response every time. Before, executing that action twice would return an error ("already favorited") on the second try.
 - Mastodon API: Limit timeline requests to 3 per timeline per 500ms per user/ip by default.
+- Admin API: `PATCH /api/pleroma/admin/users/:nickname/credentials` and `GET /api/pleroma/admin/users/:nickname/credentials`
 </details>
 
 ### Added

COPYING

@@ -1,4 +1,4 @@
-Unless otherwise stated this repository is copyright © 2017-2019
+Unless otherwise stated this repository is copyright © 2017-2020
 Pleroma Authors <https://pleroma.social/>, and is distributed under
 The GNU Affero General Public License Version 3, you should have received a
 copy of the license file as AGPL-3.
@@ -23,7 +23,7 @@ priv/static/images/pleroma-fox-tan-shy.png
 
 ---
 
-The following files are copyright © 2017-2019 Pleroma Authors
+The following files are copyright © 2017-2020 Pleroma Authors
 <https://pleroma.social/>, and are distributed under the Creative Commons
 Attribution-ShareAlike 4.0 International license, you should have received
 a copy of the license file as CC-BY-SA-4.0.

Dockerfile

@@ -12,7 +12,7 @@ RUN apk add git gcc g++ musl-dev make &&\
 	mkdir release &&\
 	mix release --path release
 
-FROM alpine:3.9
+FROM alpine:3.11
 
 ARG BUILD_DATE
 ARG VCS_REF
@@ -33,7 +33,7 @@ ARG DATA=/var/lib/pleroma
 
 RUN echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories &&\
 	apk update &&\
-	apk add ncurses postgresql-client &&\
+	apk add imagemagick ncurses postgresql-client &&\
 	adduser --system --shell /bin/false --home ${HOME} pleroma &&\
 	mkdir -p ${DATA}/uploads &&\
 	mkdir -p ${DATA}/static &&\

benchmarks/load_testing/activities.ex

@@ -0,0 +1,557 @@
+defmodule Pleroma.LoadTesting.Activities do
+  @moduledoc """
+  Module for generating different activities.
+  """
+  import Ecto.Query
+  import Pleroma.LoadTesting.Helper, only: [to_sec: 1]
+
+  alias Ecto.UUID
+  alias Pleroma.Constants
+  alias Pleroma.LoadTesting.Users
+  alias Pleroma.Repo
+  alias Pleroma.Web.CommonAPI
+
+  require Constants
+
+  @defaults [
+    iterations: 170,
+    friends_used: 20,
+    non_friends_used: 20
+  ]
+
+  @max_concurrency 10
+
+  @visibility ~w(public private direct unlisted)
+  @types ~w(simple emoji mentions hell_thread attachment tag like reblog simple_thread remote)
+  @groups ~w(user friends non_friends)
+
+  @spec generate(User.t(), keyword()) :: :ok
+  def generate(user, opts \\ []) do
+    {:ok, _} =
+      Agent.start_link(fn -> %{} end,
+        name: :benchmark_state
+      )
+
+    opts = Keyword.merge(@defaults, opts)
+
+    friends =
+      user
+      |> Users.get_users(limit: opts[:friends_used], local: :local, friends?: true)
+      |> Enum.shuffle()
+
+    non_friends =
+      user
+      |> Users.get_users(limit: opts[:non_friends_used], local: :local, friends?: false)
+      |> Enum.shuffle()
+
+    task_data =
+      for visibility <- @visibility,
+          type <- @types,
+          group <- @groups,
+          do: {visibility, type, group}
+
+    IO.puts("Starting generating #{opts[:iterations]} iterations of activities...")
+
+    friends_thread = Enum.take(friends, 5)
+    non_friends_thread = Enum.take(friends, 5)
+
+    public_long_thread = fn ->
+      generate_long_thread("public", user, friends_thread, non_friends_thread, opts)
+    end
+
+    private_long_thread = fn ->
+      generate_long_thread("private", user, friends_thread, non_friends_thread, opts)
+    end
+
+    iterations = opts[:iterations]
+
+    {time, _} =
+      :timer.tc(fn ->
+        Enum.each(
+          1..iterations,
+          fn
+            i when i == iterations - 2 ->
+              spawn(public_long_thread)
+              spawn(private_long_thread)
+              generate_activities(user, friends, non_friends, Enum.shuffle(task_data), opts)
+
+            _ ->
+              generate_activities(user, friends, non_friends, Enum.shuffle(task_data), opts)
+          end
+        )
+      end)
+
+    IO.puts("Generating iterations of activities took #{to_sec(time)} sec.\n")
+    :ok
+  end
+
+  def generate_power_intervals(opts \\ []) do
+    count = Keyword.get(opts, :count, 20)
+    power = Keyword.get(opts, :power, 2)
+    IO.puts("Generating #{count} intervals for a power #{power} series...")
+    counts = Enum.map(1..count, fn n -> :math.pow(n, power) end)
+    sum = Enum.sum(counts)
+
+    densities =
+      Enum.map(counts, fn c ->
+        c / sum
+      end)
+
+    densities
+    |> Enum.reduce(0, fn density, acc ->
+      if acc == 0 do
+        [{0, density}]
+      else
+        [{_, lower} | _] = acc
+        [{lower, lower + density} | acc]
+      end
+    end)
+    |> Enum.reverse()
+  end
+
+  def generate_tagged_activities(opts \\ []) do
+    tag_count = Keyword.get(opts, :tag_count, 20)
+    users = Keyword.get(opts, :users, Repo.all(Pleroma.User))
+    activity_count = Keyword.get(opts, :count, 200_000)
+
+    intervals = generate_power_intervals(count: tag_count)
+
+    IO.puts(
+      "Generating #{activity_count} activities using #{tag_count} different tags of format `tag_n`, starting at tag_0"
+    )
+
+    Enum.each(1..activity_count, fn _ ->
+      random = :rand.uniform()
+      i = Enum.find_index(intervals, fn {lower, upper} -> lower <= random && upper > random end)
+      CommonAPI.post(Enum.random(users), %{"status" => "a post with the tag #tag_#{i}"})
+    end)
+  end
+
+  defp generate_long_thread(visibility, user, friends, non_friends, _opts) do
+    group =
+      if visibility == "public",
+        do: "friends",
+        else: "user"
+
+    tasks = get_reply_tasks(visibility, group) |> Stream.cycle() |> Enum.take(50)
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{
+        "status" => "Start of #{visibility} long thread",
+        "visibility" => visibility
+      })
+
+    Agent.update(:benchmark_state, fn state ->
+      key =
+        if visibility == "public",
+          do: :public_thread,
+          else: :private_thread
+
+      Map.put(state, key, activity)
+    end)
+
+    acc = {activity.id, ["@" <> user.nickname, "reply to long thread"]}
+    insert_replies_for_long_thread(tasks, visibility, user, friends, non_friends, acc)
+    IO.puts("Generating #{visibility} long thread ended\n")
+  end
+
+  defp insert_replies_for_long_thread(tasks, visibility, user, friends, non_friends, acc) do
+    Enum.reduce(tasks, acc, fn
+      "friend", {id, data} ->
+        friend = Enum.random(friends)
+        insert_reply(friend, List.delete(data, "@" <> friend.nickname), id, visibility)
+
+      "non_friend", {id, data} ->
+        non_friend = Enum.random(non_friends)
+        insert_reply(non_friend, List.delete(data, "@" <> non_friend.nickname), id, visibility)
+
+      "user", {id, data} ->
+        insert_reply(user, List.delete(data, "@" <> user.nickname), id, visibility)
+    end)
+  end
+
+  defp generate_activities(user, friends, non_friends, task_data, opts) do
+    Task.async_stream(
+      task_data,
+      fn {visibility, type, group} ->
+        insert_activity(type, visibility, group, user, friends, non_friends, opts)
+      end,
+      max_concurrency: @max_concurrency,
+      timeout: 30_000
+    )
+    |> Stream.run()
+  end
+
+  defp insert_activity("simple", visibility, group, user, friends, non_friends, _opts) do
+    {:ok, _activity} =
+      group
+      |> get_actor(user, friends, non_friends)
+      |> CommonAPI.post(%{"status" => "Simple status", "visibility" => visibility})
+  end
+
+  defp insert_activity("emoji", visibility, group, user, friends, non_friends, _opts) do
+    {:ok, _activity} =
+      group
+      |> get_actor(user, friends, non_friends)
+      |> CommonAPI.post(%{
+        "status" => "Simple status with emoji :firefox:",
+        "visibility" => visibility
+      })
+  end
+
+  defp insert_activity("mentions", visibility, group, user, friends, non_friends, _opts) do
+    user_mentions =
+      get_random_mentions(friends, Enum.random(0..3)) ++
+        get_random_mentions(non_friends, Enum.random(0..3))
+
+    user_mentions =
+      if Enum.random([true, false]),
+        do: ["@" <> user.nickname | user_mentions],
+        else: user_mentions
+
+    {:ok, _activity} =
+      group
+      |> get_actor(user, friends, non_friends)
+      |> CommonAPI.post(%{
+        "status" => Enum.join(user_mentions, ", ") <> " simple status with mentions",
+        "visibility" => visibility
+      })
+  end
+
+  defp insert_activity("hell_thread", visibility, group, user, friends, non_friends, _opts) do
+    mentions =
+      with {:ok, nil} <- Cachex.get(:user_cache, "hell_thread_mentions") do
+        cached =
+          ([user | Enum.take(friends, 10)] ++ Enum.take(non_friends, 10))
+          |> Enum.map(&"@#{&1.nickname}")
+          |> Enum.join(", ")
+
+        Cachex.put(:user_cache, "hell_thread_mentions", cached)
+        cached
+      else
+        {:ok, cached} -> cached
+      end
+
+    {:ok, _activity} =
+      group
+      |> get_actor(user, friends, non_friends)
+      |> CommonAPI.post(%{
+        "status" => mentions <> " hell thread status",
+        "visibility" => visibility
+      })
+  end
+
+  defp insert_activity("attachment", visibility, group, user, friends, non_friends, _opts) do
+    actor = get_actor(group, user, friends, non_friends)
+
+    obj_data = %{
+      "actor" => actor.ap_id,
+      "name" => "4467-11.jpg",
+      "type" => "Document",
+      "url" => [
+        %{
+          "href" =>
+            "#{Pleroma.Web.base_url()}/media/b1b873552422a07bf53af01f3c231c841db4dfc42c35efde681abaf0f2a4eab7.jpg",
+          "mediaType" => "image/jpeg",
+          "type" => "Link"
+        }
+      ]
+    }
+
+    object = Repo.insert!(%Pleroma.Object{data: obj_data})
+
+    {:ok, _activity} =
+      CommonAPI.post(actor, %{
+        "status" => "Post with attachment",
+        "visibility" => visibility,
+        "media_ids" => [object.id]
+      })
+  end
+
+  defp insert_activity("tag", visibility, group, user, friends, non_friends, _opts) do
+    {:ok, _activity} =
+      group
+      |> get_actor(user, friends, non_friends)
+      |> CommonAPI.post(%{"status" => "Status with #tag", "visibility" => visibility})
+  end
+
+  defp insert_activity("like", visibility, group, user, friends, non_friends, opts) do
+    actor = get_actor(group, user, friends, non_friends)
+
+    with activity_id when not is_nil(activity_id) <- get_random_create_activity_id(),
+         {:ok, _activity} <- CommonAPI.favorite(actor, activity_id) do
+      :ok
+    else
+      {:error, _} ->
+        insert_activity("like", visibility, group, user, friends, non_friends, opts)
+
+      nil ->
+        Process.sleep(15)
+        insert_activity("like", visibility, group, user, friends, non_friends, opts)
+    end
+  end
+
+  defp insert_activity("reblog", visibility, group, user, friends, non_friends, opts) do
+    actor = get_actor(group, user, friends, non_friends)
+
+    with activity_id when not is_nil(activity_id) <- get_random_create_activity_id(),
+         {:ok, _activity, _object} <- CommonAPI.repeat(activity_id, actor) do
+      :ok
+    else
+      {:error, _} ->
+        insert_activity("reblog", visibility, group, user, friends, non_friends, opts)
+
+      nil ->
+        Process.sleep(15)
+        insert_activity("reblog", visibility, group, user, friends, non_friends, opts)
+    end
+  end
+
+  defp insert_activity("simple_thread", visibility, group, user, friends, non_friends, _opts)
+       when visibility in ["public", "unlisted", "private"] do
+    actor = get_actor(group, user, friends, non_friends)
+    tasks = get_reply_tasks(visibility, group)
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{"status" => "Simple status", "visibility" => visibility})
+
+    acc = {activity.id, ["@" <> actor.nickname, "reply to status"]}
+    insert_replies(tasks, visibility, user, friends, non_friends, acc)
+  end
+
+  defp insert_activity("simple_thread", "direct", group, user, friends, non_friends, _opts) do
+    actor = get_actor(group, user, friends, non_friends)
+    tasks = get_reply_tasks("direct", group)
+
+    list =
+      case group do
+        "non_friends" ->
+          Enum.take(non_friends, 3)
+
+        _ ->
+          Enum.take(friends, 3)
+      end
+
+    data = Enum.map(list, &("@" <> &1.nickname))
+
+    {:ok, activity} =
+      CommonAPI.post(actor, %{
+        "status" => Enum.join(data, ", ") <> "simple status",
+        "visibility" => "direct"
+      })
+
+    acc = {activity.id, ["@" <> user.nickname | data] ++ ["reply to status"]}
+    insert_direct_replies(tasks, user, list, acc)
+  end
+
+  defp insert_activity("remote", _, "user", _, _, _, _), do: :ok
+
+  defp insert_activity("remote", visibility, group, user, _friends, _non_friends, opts) do
+    remote_friends =
+      Users.get_users(user, limit: opts[:friends_used], local: :external, friends?: true)
+
+    remote_non_friends =
+      Users.get_users(user, limit: opts[:non_friends_used], local: :external, friends?: false)
+
+    actor = get_actor(group, user, remote_friends, remote_non_friends)
+
+    {act_data, obj_data} = prepare_activity_data(actor, visibility, user)
+    {activity_data, object_data} = other_data(actor)
+
+    activity_data
+    |> Map.merge(act_data)
+    |> Map.put("object", Map.merge(object_data, obj_data))
+    |> Pleroma.Web.ActivityPub.ActivityPub.insert(false)
+  end
+
+  defp get_actor("user", user, _friends, _non_friends), do: user
+  defp get_actor("friends", _user, friends, _non_friends), do: Enum.random(friends)
+  defp get_actor("non_friends", _user, _friends, non_friends), do: Enum.random(non_friends)
+
+  defp other_data(actor) do
+    %{host: host} = URI.parse(actor.ap_id)
+    datetime = DateTime.utc_now()
+    context_id = "http://#{host}:4000/contexts/#{UUID.generate()}"
+    activity_id = "http://#{host}:4000/activities/#{UUID.generate()}"
+    object_id = "http://#{host}:4000/objects/#{UUID.generate()}"
+
+    activity_data = %{
+      "actor" => actor.ap_id,
+      "context" => context_id,
+      "id" => activity_id,
+      "published" => datetime,
+      "type" => "Create",
+      "directMessage" => false
+    }
+
+    object_data = %{
+      "actor" => actor.ap_id,
+      "attachment" => [],
+      "attributedTo" => actor.ap_id,
+      "bcc" => [],
+      "bto" => [],
+      "content" => "Remote post",
+      "context" => context_id,
+      "conversation" => context_id,
+      "emoji" => %{},
+      "id" => object_id,
+      "published" => datetime,
+      "sensitive" => false,
+      "summary" => "",
+      "tag" => [],
+      "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+      "type" => "Note"
+    }
+
+    {activity_data, object_data}
+  end
+
+  defp prepare_activity_data(actor, "public", _mention) do
+    obj_data = %{
+      "cc" => [actor.follower_address],
+      "to" => [Constants.as_public()]
+    }
+
+    act_data = %{
+      "cc" => [actor.follower_address],
+      "to" => [Constants.as_public()]
+    }
+
+    {act_data, obj_data}
+  end
+
+  defp prepare_activity_data(actor, "private", _mention) do
+    obj_data = %{
+      "cc" => [],
+      "to" => [actor.follower_address]
+    }
+
+    act_data = %{
+      "cc" => [],
+      "to" => [actor.follower_address]
+    }
+
+    {act_data, obj_data}
+  end
+
+  defp prepare_activity_data(actor, "unlisted", _mention) do
+    obj_data = %{
+      "cc" => [Constants.as_public()],
+      "to" => [actor.follower_address]
+    }
+
+    act_data = %{
+      "cc" => [Constants.as_public()],
+      "to" => [actor.follower_address]
+    }
+
+    {act_data, obj_data}
+  end
+
+  defp prepare_activity_data(_actor, "direct", mention) do
+    %{host: mentioned_host} = URI.parse(mention.ap_id)
+
+    obj_data = %{
+      "cc" => [],
+      "content" =>
+        "<span class=\"h-card\"><a class=\"u-url mention\" href=\"#{mention.ap_id}\" rel=\"ugc\">@<span>#{
+          mention.nickname
+        }</span></a></span> direct message",
+      "tag" => [
+        %{
+          "href" => mention.ap_id,
+          "name" => "@#{mention.nickname}@#{mentioned_host}",
+          "type" => "Mention"
+        }
+      ],
+      "to" => [mention.ap_id]
+    }
+
+    act_data = %{
+      "cc" => [],
+      "directMessage" => true,
+      "to" => [mention.ap_id]
+    }
+
+    {act_data, obj_data}
+  end
+
+  defp get_reply_tasks("public", "user"), do: ~w(friend non_friend user)
+  defp get_reply_tasks("public", "friends"), do: ~w(non_friend user friend)
+  defp get_reply_tasks("public", "non_friends"), do: ~w(user friend non_friend)
+
+  defp get_reply_tasks(visibility, "user") when visibility in ["unlisted", "private"],
+    do: ~w(friend user friend)
+
+  defp get_reply_tasks(visibility, "friends") when visibility in ["unlisted", "private"],
+    do: ~w(user friend user)
+
+  defp get_reply_tasks(visibility, "non_friends") when visibility in ["unlisted", "private"],
+    do: []
+
+  defp get_reply_tasks("direct", "user"), do: ~w(friend user friend)
+  defp get_reply_tasks("direct", "friends"), do: ~w(user friend user)
+  defp get_reply_tasks("direct", "non_friends"), do: ~w(user non_friend user)
+
+  defp insert_replies(tasks, visibility, user, friends, non_friends, acc) do
+    Enum.reduce(tasks, acc, fn
+      "friend", {id, data} ->
+        friend = Enum.random(friends)
+        insert_reply(friend, data, id, visibility)
+
+      "non_friend", {id, data} ->
+        non_friend = Enum.random(non_friends)
+        insert_reply(non_friend, data, id, visibility)
+
+      "user", {id, data} ->
+        insert_reply(user, data, id, visibility)
+    end)
+  end
+
+  defp insert_direct_replies(tasks, user, list, acc) do
+    Enum.reduce(tasks, acc, fn
+      group, {id, data} when group in ["friend", "non_friend"] ->
+        actor = Enum.random(list)
+
+        {reply_id, _} =
+          insert_reply(actor, List.delete(data, "@" <> actor.nickname), id, "direct")
+
+        {reply_id, data}
+
+      "user", {id, data} ->
+        {reply_id, _} = insert_reply(user, List.delete(data, "@" <> user.nickname), id, "direct")
+        {reply_id, data}
+    end)
+  end
+
+  defp insert_reply(actor, data, activity_id, visibility) do
+    {:ok, reply} =
+      CommonAPI.post(actor, %{
+        "status" => Enum.join(data, ", "),
+        "visibility" => visibility,
+        "in_reply_to_status_id" => activity_id
+      })
+
+    {reply.id, ["@" <> actor.nickname | data]}
+  end
+
+  defp get_random_mentions(_users, count) when count == 0, do: []
+
+  defp get_random_mentions(users, count) do
+    users
+    |> Enum.shuffle()
+    |> Enum.take(count)
+    |> Enum.map(&"@#{&1.nickname}")
+  end
+
+  defp get_random_create_activity_id do
+    Repo.one(
+      from(a in Pleroma.Activity,
+        where: fragment("(?)->>'type' = ?", a.data, ^"Create"),
+        order_by: fragment("RANDOM()"),
+        limit: 1,
+        select: a.id
+      )
+    )
+  end
+end

benchmarks/load_testing/fetcher.ex

@@ -1,260 +1,553 @@
 defmodule Pleroma.LoadTesting.Fetcher do
-  use Pleroma.LoadTesting.Helper
+  alias Pleroma.Activity
+  alias Pleroma.Pagination
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.MastodonAPI.MastodonAPI
+  alias Pleroma.Web.MastodonAPI.StatusView
 
-  def fetch_user(user) do
+  @spec run_benchmarks(User.t()) :: any()
-    Benchee.run(%{
+  def run_benchmarks(user) do
-      "By id" => fn -> Repo.get_by(User, id: user.id) end,
+    fetch_user(user)
-      "By ap_id" => fn -> Repo.get_by(User, ap_id: user.ap_id) end,
+    fetch_timelines(user)
-      "By email" => fn -> Repo.get_by(User, email: user.email) end,
+    render_views(user)
-      "By nickname" => fn -> Repo.get_by(User, nickname: user.nickname) end
-    })
   end
 
-  def query_timelines(user) do
+  defp formatters do
-    home_timeline_params = %{
+    [
-      "count" => 20,
+      Benchee.Formatters.Console
-      "with_muted" => true,
+    ]
-      "type" => ["Create", "Announce"],
+  end
+
+  defp fetch_user(user) do
+    Benchee.run(
+      %{
+        "By id" => fn -> Repo.get_by(User, id: user.id) end,
+        "By ap_id" => fn -> Repo.get_by(User, ap_id: user.ap_id) end,
+        "By email" => fn -> Repo.get_by(User, email: user.email) end,
+        "By nickname" => fn -> Repo.get_by(User, nickname: user.nickname) end
+      },
+      formatters: formatters()
+    )
+  end
+
+  defp fetch_timelines(user) do
+    fetch_home_timeline(user)
+    fetch_direct_timeline(user)
+    fetch_public_timeline(user)
+    fetch_public_timeline(user, :local)
+    fetch_public_timeline(user, :tag)
+    fetch_notifications(user)
+    fetch_favourites(user)
+    fetch_long_thread(user)
+    fetch_timelines_with_reply_filtering(user)
+  end
+
+  defp render_views(user) do
+    render_timelines(user)
+    render_long_thread(user)
+  end
+
+  defp opts_for_home_timeline(user) do
+    %{
       "blocking_user" => user,
+      "count" => "20",
       "muting_user" => user,
+      "type" => ["Create", "Announce"],
+      "user" => user,
+      "with_muted" => "true"
+    }
+  end
+
+  defp fetch_home_timeline(user) do
+    opts = opts_for_home_timeline(user)
+
+    recipients = [user.ap_id | User.following(user)]
+
+    first_page_last =
+      ActivityPub.fetch_activities(recipients, opts) |> Enum.reverse() |> List.last()
+
+    second_page_last =
+      ActivityPub.fetch_activities(recipients, Map.put(opts, "max_id", first_page_last.id))
+      |> Enum.reverse()
+      |> List.last()
+
+    third_page_last =
+      ActivityPub.fetch_activities(recipients, Map.put(opts, "max_id", second_page_last.id))
+      |> Enum.reverse()
+      |> List.last()
+
+    forth_page_last =
+      ActivityPub.fetch_activities(recipients, Map.put(opts, "max_id", third_page_last.id))
+      |> Enum.reverse()
+      |> List.last()
+
+    Benchee.run(
+      %{
+        "home timeline" => fn opts -> ActivityPub.fetch_activities(recipients, opts) end
+      },
+      inputs: %{
+        "1 page" => opts,
+        "2 page" => Map.put(opts, "max_id", first_page_last.id),
+        "3 page" => Map.put(opts, "max_id", second_page_last.id),
+        "4 page" => Map.put(opts, "max_id", third_page_last.id),
+        "5 page" => Map.put(opts, "max_id", forth_page_last.id),
+        "1 page only media" => Map.put(opts, "only_media", "true"),
+        "2 page only media" =>
+          Map.put(opts, "max_id", first_page_last.id) |> Map.put("only_media", "true"),
+        "3 page only media" =>
+          Map.put(opts, "max_id", second_page_last.id) |> Map.put("only_media", "true"),
+        "4 page only media" =>
+          Map.put(opts, "max_id", third_page_last.id) |> Map.put("only_media", "true"),
+        "5 page only media" =>
+          Map.put(opts, "max_id", forth_page_last.id) |> Map.put("only_media", "true")
+      },
+      formatters: formatters()
+    )
+  end
+
+  defp opts_for_direct_timeline(user) do
+    %{
+      :visibility => "direct",
+      "blocking_user" => user,
+      "count" => "20",
+      "type" => "Create",
+      "user" => user,
+      "with_muted" => "true"
+    }
+  end
+
+  defp fetch_direct_timeline(user) do
+    recipients = [user.ap_id]
+
+    opts = opts_for_direct_timeline(user)
+
+    first_page_last =
+      recipients
+      |> ActivityPub.fetch_activities_query(opts)
+      |> Pagination.fetch_paginated(opts)
+      |> List.last()
+
+    opts2 = Map.put(opts, "max_id", first_page_last.id)
+
+    second_page_last =
+      recipients
+      |> ActivityPub.fetch_activities_query(opts2)
+      |> Pagination.fetch_paginated(opts2)
+      |> List.last()
+
+    opts3 = Map.put(opts, "max_id", second_page_last.id)
+
+    third_page_last =
+      recipients
+      |> ActivityPub.fetch_activities_query(opts3)
+      |> Pagination.fetch_paginated(opts3)
+      |> List.last()
+
+    opts4 = Map.put(opts, "max_id", third_page_last.id)
+
+    forth_page_last =
+      recipients
+      |> ActivityPub.fetch_activities_query(opts4)
+      |> Pagination.fetch_paginated(opts4)
+      |> List.last()
+
+    Benchee.run(
+      %{
+        "direct timeline" => fn opts ->
+          ActivityPub.fetch_activities_query(recipients, opts) |> Pagination.fetch_paginated(opts)
+        end
+      },
+      inputs: %{
+        "1 page" => opts,
+        "2 page" => opts2,
+        "3 page" => opts3,
+        "4 page" => opts4,
+        "5 page" => Map.put(opts4, "max_id", forth_page_last.id)
+      },
+      formatters: formatters()
+    )
+  end
+
+  defp opts_for_public_timeline(user) do
+    %{
+      "type" => ["Create", "Announce"],
+      "local_only" => false,
+      "blocking_user" => user,
+      "muting_user" => user
+    }
+  end
+
+  defp opts_for_public_timeline(user, :local) do
+    %{
+      "type" => ["Create", "Announce"],
+      "local_only" => true,
+      "blocking_user" => user,
+      "muting_user" => user
+    }
+  end
+
+  defp opts_for_public_timeline(user, :tag) do
+    %{
+      "blocking_user" => user,
+      "count" => "20",
+      "local_only" => nil,
+      "muting_user" => user,
+      "tag" => ["tag"],
+      "tag_all" => [],
+      "tag_reject" => [],
+      "type" => "Create",
+      "user" => user,
+      "with_muted" => "true"
+    }
+  end
+
+  defp fetch_public_timeline(user) do
+    opts = opts_for_public_timeline(user)
+
+    fetch_public_timeline(opts, "public timeline")
+  end
+
+  defp fetch_public_timeline(user, :local) do
+    opts = opts_for_public_timeline(user, :local)
+
+    fetch_public_timeline(opts, "public timeline only local")
+  end
+
+  defp fetch_public_timeline(user, :tag) do
+    opts = opts_for_public_timeline(user, :tag)
+
+    fetch_public_timeline(opts, "hashtag timeline")
+  end
+
+  defp fetch_public_timeline(user, :only_media) do
+    opts = opts_for_public_timeline(user) |> Map.put("only_media", "true")
+
+    fetch_public_timeline(opts, "public timeline only media")
+  end
+
+  defp fetch_public_timeline(opts, title) when is_binary(title) do
+    first_page_last = ActivityPub.fetch_public_activities(opts) |> List.last()
+
+    second_page_last =
+      ActivityPub.fetch_public_activities(Map.put(opts, "max_id", first_page_last.id))
+      |> List.last()
+
+    third_page_last =
+      ActivityPub.fetch_public_activities(Map.put(opts, "max_id", second_page_last.id))
+      |> List.last()
+
+    forth_page_last =
+      ActivityPub.fetch_public_activities(Map.put(opts, "max_id", third_page_last.id))
+      |> List.last()
+
+    Benchee.run(
+      %{
+        title => fn opts ->
+          ActivityPub.fetch_public_activities(opts)
+        end
+      },
+      inputs: %{
+        "1 page" => opts,
+        "2 page" => Map.put(opts, "max_id", first_page_last.id),
+        "3 page" => Map.put(opts, "max_id", second_page_last.id),
+        "4 page" => Map.put(opts, "max_id", third_page_last.id),
+        "5 page" => Map.put(opts, "max_id", forth_page_last.id)
+      },
+      formatters: formatters()
+    )
+  end
+
+  defp opts_for_notifications do
+    %{"count" => "20", "with_muted" => "true"}
+  end
+
+  defp fetch_notifications(user) do
+    opts = opts_for_notifications()
+
+    first_page_last = MastodonAPI.get_notifications(user, opts) |> List.last()
+
+    second_page_last =
+      MastodonAPI.get_notifications(user, Map.put(opts, "max_id", first_page_last.id))
+      |> List.last()
+
+    third_page_last =
+      MastodonAPI.get_notifications(user, Map.put(opts, "max_id", second_page_last.id))
+      |> List.last()
+
+    forth_page_last =
+      MastodonAPI.get_notifications(user, Map.put(opts, "max_id", third_page_last.id))
+      |> List.last()
+
+    Benchee.run(
+      %{
+        "Notifications" => fn opts ->
+          MastodonAPI.get_notifications(user, opts)
+        end
+      },
+      inputs: %{
+        "1 page" => opts,
+        "2 page" => Map.put(opts, "max_id", first_page_last.id),
+        "3 page" => Map.put(opts, "max_id", second_page_last.id),
+        "4 page" => Map.put(opts, "max_id", third_page_last.id),
+        "5 page" => Map.put(opts, "max_id", forth_page_last.id)
+      },
+      formatters: formatters()
+    )
+  end
+
+  defp fetch_favourites(user) do
+    first_page_last = ActivityPub.fetch_favourites(user) |> List.last()
+
+    second_page_last =
+      ActivityPub.fetch_favourites(user, %{"max_id" => first_page_last.id}) |> List.last()
+
+    third_page_last =
+      ActivityPub.fetch_favourites(user, %{"max_id" => second_page_last.id}) |> List.last()
+
+    forth_page_last =
+      ActivityPub.fetch_favourites(user, %{"max_id" => third_page_last.id}) |> List.last()
+
+    Benchee.run(
+      %{
+        "Favourites" => fn opts ->
+          ActivityPub.fetch_favourites(user, opts)
+        end
+      },
+      inputs: %{
+        "1 page" => %{},
+        "2 page" => %{"max_id" => first_page_last.id},
+        "3 page" => %{"max_id" => second_page_last.id},
+        "4 page" => %{"max_id" => third_page_last.id},
+        "5 page" => %{"max_id" => forth_page_last.id}
+      },
+      formatters: formatters()
+    )
+  end
+
+  defp opts_for_long_thread(user) do
+    %{
+      "blocking_user" => user,
       "user" => user
     }
+  end
 
-    mastodon_public_timeline_params = %{
+  defp fetch_long_thread(user) do
-      "count" => 20,
+    %{public_thread: public, private_thread: private} =
-      "local_only" => true,
+      Agent.get(:benchmark_state, fn state -> state end)
-      "only_media" => "false",
-      "type" => ["Create", "Announce"],
-      "with_muted" => "true",
-      "blocking_user" => user,
-      "muting_user" => user
-    }
 
-    mastodon_federated_timeline_params = %{
+    opts = opts_for_long_thread(user)
-      "count" => 20,
-      "only_media" => "false",
-      "type" => ["Create", "Announce"],
-      "with_muted" => "true",
-      "blocking_user" => user,
-      "muting_user" => user
-    }
 
-    following = User.following(user)
+    private_input = {private.data["context"], Map.put(opts, "exclude_id", private.id)}
 
-    Benchee.run(%{
+    public_input = {public.data["context"], Map.put(opts, "exclude_id", public.id)}
-      "User home timeline" => fn ->
-        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities(
-          following,
-          home_timeline_params
-        )
-      end,
-      "User mastodon public timeline" => fn ->
-        Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
-          mastodon_public_timeline_params
-        )
-      end,
-      "User mastodon federated public timeline" => fn ->
-        Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
-          mastodon_federated_timeline_params
-        )
-      end
-    })
 
-    home_activities =
+    Benchee.run(
-      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities(
+      %{
-        following,
+        "fetch context" => fn {context, opts} ->
-        home_timeline_params
+          ActivityPub.fetch_activities_for_context(context, opts)
+        end
+      },
+      inputs: %{
+        "Private long thread" => private_input,
+        "Public long thread" => public_input
+      },
+      formatters: formatters()
+    )
+  end
+
+  defp render_timelines(user) do
+    opts = opts_for_home_timeline(user)
+
+    recipients = [user.ap_id | User.following(user)]
+
+    home_activities = ActivityPub.fetch_activities(recipients, opts) |> Enum.reverse()
+
+    recipients = [user.ap_id]
+
+    opts = opts_for_direct_timeline(user)
+
+    direct_activities =
+      recipients
+      |> ActivityPub.fetch_activities_query(opts)
+      |> Pagination.fetch_paginated(opts)
+
+    opts = opts_for_public_timeline(user)
+
+    public_activities = ActivityPub.fetch_public_activities(opts)
+
+    opts = opts_for_public_timeline(user, :tag)
+
+    tag_activities = ActivityPub.fetch_public_activities(opts)
+
+    opts = opts_for_notifications()
+
+    notifications = MastodonAPI.get_notifications(user, opts)
+
+    favourites = ActivityPub.fetch_favourites(user)
+
+    output_relationships =
+      !!Pleroma.Config.get([:extensions, :output_relationships_in_statuses_by_default])
+
+    Benchee.run(
+      %{
+        "Rendering home timeline" => fn ->
+          StatusView.render("index.json", %{
+            activities: home_activities,
+            for: user,
+            as: :activity,
+            skip_relationships: !output_relationships
+          })
+        end,
+        "Rendering direct timeline" => fn ->
+          StatusView.render("index.json", %{
+            activities: direct_activities,
+            for: user,
+            as: :activity,
+            skip_relationships: !output_relationships
+          })
+        end,
+        "Rendering public timeline" => fn ->
+          StatusView.render("index.json", %{
+            activities: public_activities,
+            for: user,
+            as: :activity,
+            skip_relationships: !output_relationships
+          })
+        end,
+        "Rendering tag timeline" => fn ->
+          StatusView.render("index.json", %{
+            activities: tag_activities,
+            for: user,
+            as: :activity,
+            skip_relationships: !output_relationships
+          })
+        end,
+        "Rendering notifications" => fn ->
+          Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{
+            notifications: notifications,
+            for: user,
+            skip_relationships: !output_relationships
+          })
+        end,
+        "Rendering favourites timeline" => fn ->
+          StatusView.render("index.json", %{
+            activities: favourites,
+            for: user,
+            as: :activity,
+            skip_relationships: !output_relationships
+          })
+        end
+      },
+      formatters: formatters()
+    )
+  end
+
+  defp render_long_thread(user) do
+    %{public_thread: public, private_thread: private} =
+      Agent.get(:benchmark_state, fn state -> state end)
+
+    opts = %{for: user}
+    public_activity = Activity.get_by_id_with_object(public.id)
+    private_activity = Activity.get_by_id_with_object(private.id)
+
+    Benchee.run(
+      %{
+        "render" => fn opts ->
+          StatusView.render("show.json", opts)
+        end
+      },
+      inputs: %{
+        "Public root" => Map.put(opts, :activity, public_activity),
+        "Private root" => Map.put(opts, :activity, private_activity)
+      },
+      formatters: formatters()
+    )
+
+    fetch_opts = opts_for_long_thread(user)
+
+    public_context =
+      ActivityPub.fetch_activities_for_context(
+        public.data["context"],
+        Map.put(fetch_opts, "exclude_id", public.id)
       )
 
-    public_activities =
+    private_context =
-      Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params)
+      ActivityPub.fetch_activities_for_context(
-
+        private.data["context"],
-    public_federated_activities =
+        Map.put(fetch_opts, "exclude_id", private.id)
-      Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
-        mastodon_federated_timeline_params
       )
 
-    Benchee.run(%{
+    Benchee.run(
-      "Rendering home timeline" => fn ->
+      %{
-        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+        "render" => fn opts ->
-          activities: home_activities,
+          StatusView.render("context.json", opts)
-          for: user,
+        end
-          as: :activity
+      },
-        })
+      inputs: %{
-      end,
+        "Public context" => %{user: user, activity: public_activity, activities: public_context},
-      "Rendering public timeline" => fn ->
+        "Private context" => %{
-        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          user: user,
-          activities: public_activities,
+          activity: private_activity,
-          for: user,
+          activities: private_context
-          as: :activity
-        })
-      end,
-      "Rendering public federated timeline" => fn ->
-        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
-          activities: public_federated_activities,
-          for: user,
-          as: :activity
-        })
-      end,
-      "Rendering favorites timeline" => fn ->
-        conn = Phoenix.ConnTest.build_conn(:get, "http://localhost:4001/api/v1/favourites", nil)
-        Pleroma.Web.MastodonAPI.StatusController.favourites(
-          %Plug.Conn{conn |
-                     assigns: %{user: user},
-                     query_params:  %{"limit" => "0"},
-                     body_params: %{},
-                     cookies: %{},
-                     params: %{},
-                     path_params: %{},
-                     private: %{
-                       Pleroma.Web.Router => {[], %{}},
-                       phoenix_router: Pleroma.Web.Router,
-                       phoenix_action: :favourites,
-                       phoenix_controller: Pleroma.Web.MastodonAPI.StatusController,
-                       phoenix_endpoint: Pleroma.Web.Endpoint,
-                       phoenix_format: "json",
-                       phoenix_layout: {Pleroma.Web.LayoutView, "app.html"},
-                       phoenix_recycled: true,
-
-                       phoenix_view: Pleroma.Web.MastodonAPI.StatusView,
-                       plug_session: %{"user_id" => user.id},
-                       plug_session_fetch: :done,
-                       plug_session_info: :write,
-                       plug_skip_csrf_protection: true
-                     }
-          },
-          %{})
-      end,
-    })
-  end
-
-  def query_notifications(user) do
-    without_muted_params = %{"count" => "20", "with_muted" => "false"}
-    with_muted_params = %{"count" => "20", "with_muted" => "true"}
-
-    Benchee.run(%{
-      "Notifications without muted" => fn ->
-        Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params)
-      end,
-      "Notifications with muted" => fn ->
-        Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params)
-      end
-    })
-
-    without_muted_notifications =
-      Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params)
-
-    with_muted_notifications =
-      Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params)
-
-    Benchee.run(%{
-      "Render notifications without muted" => fn ->
-        Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{
-          notifications: without_muted_notifications,
-          for: user
-        })
-      end,
-      "Render notifications with muted" => fn ->
-        Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{
-          notifications: with_muted_notifications,
-          for: user
-        })
-      end
-    })
-  end
-
-  def query_dms(user) do
-    params = %{
-      "count" => "20",
-      "with_muted" => "true",
-      "type" => "Create",
-      "blocking_user" => user,
-      "user" => user,
-      visibility: "direct"
-    }
-
-    Benchee.run(%{
-      "Direct messages with muted" => fn ->
-        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
-        |> Pleroma.Pagination.fetch_paginated(params)
-      end,
-      "Direct messages without muted" => fn ->
-        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
-        |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false))
-      end
-    })
-
-    dms_with_muted =
-      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
-      |> Pleroma.Pagination.fetch_paginated(params)
-
-    dms_without_muted =
-      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
-      |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false))
-
-    Benchee.run(%{
-      "Rendering dms with muted" => fn ->
-        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
-          activities: dms_with_muted,
-          for: user,
-          as: :activity
-        })
-      end,
-      "Rendering dms without muted" => fn ->
-        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
-          activities: dms_without_muted,
-          for: user,
-          as: :activity
-        })
-      end
-    })
-  end
-
-  def query_long_thread(user, activity) do
-    Benchee.run(%{
-      "Fetch main post" => fn ->
-        Pleroma.Activity.get_by_id_with_object(activity.id)
-      end,
-      "Fetch context of main post" => fn ->
-        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
-          activity.data["context"],
-          %{
-            "blocking_user" => user,
-            "user" => user,
-            "exclude_id" => activity.id
-          }
-        )
-      end
-    })
-
-    activity = Pleroma.Activity.get_by_id_with_object(activity.id)
-
-    context =
-      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
-        activity.data["context"],
-        %{
-          "blocking_user" => user,
-          "user" => user,
-          "exclude_id" => activity.id
         }
-      )
+      },
+      formatters: formatters()
+    )
+  end
 
-    Benchee.run(%{
+  defp fetch_timelines_with_reply_filtering(user) do
-      "Render status" => fn ->
+    public_params = opts_for_public_timeline(user)
-        Pleroma.Web.MastodonAPI.StatusView.render("show.json", %{
+
-          activity: activity,
+    Benchee.run(
-          for: user
+      %{
-        })
+        "Public timeline without reply filtering" => fn ->
-      end,
+          ActivityPub.fetch_public_activities(public_params)
-      "Render context" => fn ->
+        end,
-        Pleroma.Web.MastodonAPI.StatusView.render(
+        "Public timeline with reply filtering - following" => fn ->
-          "index.json",
+          public_params
-          for: user,
+          |> Map.put("reply_visibility", "following")
-          activities: context,
+          |> Map.put("reply_filtering_user", user)
-          as: :activity
+          |> ActivityPub.fetch_public_activities()
-        )
+        end,
-        |> Enum.reverse()
+        "Public timeline with reply filtering - self" => fn ->
-      end
+          public_params
-    })
+          |> Map.put("reply_visibility", "self")
+          |> Map.put("reply_filtering_user", user)
+          |> ActivityPub.fetch_public_activities()
+        end
+      },
+      formatters: formatters()
+    )
+
+    private_params = opts_for_home_timeline(user)
+
+    recipients = [user.ap_id | User.following(user)]
+
+    Benchee.run(
+      %{
+        "Home timeline without reply filtering" => fn ->
+          ActivityPub.fetch_activities(recipients, private_params)
+        end,
+        "Home timeline with reply filtering - following" => fn ->
+          private_params =
+            private_params
+            |> Map.put("reply_filtering_user", user)
+            |> Map.put("reply_visibility", "following")
+
+          ActivityPub.fetch_activities(recipients, private_params)
+        end,
+        "Home timeline with reply filtering - self" => fn ->
+          private_params =
+            private_params
+            |> Map.put("reply_filtering_user", user)
+            |> Map.put("reply_visibility", "self")
+
+          ActivityPub.fetch_activities(recipients, private_params)
+        end
+      },
+      formatters: formatters()
+    )
   end
 end

benchmarks/load_testing/generator.ex

@@ -1,409 +0,0 @@
-defmodule Pleroma.LoadTesting.Generator do
-  use Pleroma.LoadTesting.Helper
-  alias Pleroma.Web.CommonAPI
-
-  def generate_like_activities(user, posts) do
-    count_likes = Kernel.trunc(length(posts) / 4)
-    IO.puts("Starting generating #{count_likes} like activities...")
-
-    {time, _} =
-      :timer.tc(fn ->
-        Task.async_stream(
-          Enum.take_random(posts, count_likes),
-          fn post -> {:ok, _, _} = CommonAPI.favorite(post.id, user) end,
-          max_concurrency: 10,
-          timeout: 30_000
-        )
-        |> Stream.run()
-      end)
-
-    IO.puts("Inserting like activities take #{to_sec(time)} sec.\n")
-  end
-
-  def generate_users(opts) do
-    IO.puts("Starting generating #{opts[:users_max]} users...")
-    {time, _} = :timer.tc(fn -> do_generate_users(opts) end)
-
-    IO.puts("Inserting users take #{to_sec(time)} sec.\n")
-  end
-
-  defp do_generate_users(opts) do
-    max = Keyword.get(opts, :users_max)
-
-    Task.async_stream(
-      1..max,
-      &generate_user_data(&1),
-      max_concurrency: 10,
-      timeout: 30_000
-    )
-    |> Enum.to_list()
-  end
-
-  defp generate_user_data(i) do
-    remote = Enum.random([true, false])
-
-    user = %User{
-      name: "Test テスト User #{i}",
-      email: "user#{i}@example.com",
-      nickname: "nick#{i}",
-      password_hash:
-        "$pbkdf2-sha512$160000$bU.OSFI7H/yqWb5DPEqyjw$uKp/2rmXw12QqnRRTqTtuk2DTwZfF8VR4MYW2xMeIlqPR/UX1nT1CEKVUx2CowFMZ5JON8aDvURrZpJjSgqXrg",
-      bio: "Tester Number #{i}",
-      local: remote
-    }
-
-    user_urls =
-      if remote do
-        base_url =
-          Enum.random(["https://domain1.com", "https://domain2.com", "https://domain3.com"])
-
-        ap_id = "#{base_url}/users/#{user.nickname}"
-
-        %{
-          ap_id: ap_id,
-          follower_address: ap_id <> "/followers",
-          following_address: ap_id <> "/following"
-        }
-      else
-        %{
-          ap_id: User.ap_id(user),
-          follower_address: User.ap_followers(user),
-          following_address: User.ap_following(user)
-        }
-      end
-
-    user = Map.merge(user, user_urls)
-
-    Repo.insert!(user)
-  end
-
-  def generate_activities(user, users) do
-    do_generate_activities(user, users)
-  end
-
-  defp do_generate_activities(user, users) do
-    IO.puts("Starting generating 20000 common activities...")
-
-    {time, _} =
-      :timer.tc(fn ->
-        Task.async_stream(
-          1..20_000,
-          fn _ ->
-            do_generate_activity([user | users])
-          end,
-          max_concurrency: 10,
-          timeout: 30_000
-        )
-        |> Stream.run()
-      end)
-
-    IO.puts("Inserting common activities take #{to_sec(time)} sec.\n")
-
-    IO.puts("Starting generating 20000 activities with mentions...")
-
-    {time, _} =
-      :timer.tc(fn ->
-        Task.async_stream(
-          1..20_000,
-          fn _ ->
-            do_generate_activity_with_mention(user, users)
-          end,
-          max_concurrency: 10,
-          timeout: 30_000
-        )
-        |> Stream.run()
-      end)
-
-    IO.puts("Inserting activities with menthions take #{to_sec(time)} sec.\n")
-
-    IO.puts("Starting generating 10000 activities with threads...")
-
-    {time, _} =
-      :timer.tc(fn ->
-        Task.async_stream(
-          1..10_000,
-          fn _ ->
-            do_generate_threads([user | users])
-          end,
-          max_concurrency: 10,
-          timeout: 30_000
-        )
-        |> Stream.run()
-      end)
-
-    IO.puts("Inserting activities with threads take #{to_sec(time)} sec.\n")
-  end
-
-  defp do_generate_activity(users) do
-    post = %{
-      "status" => "Some status without mention with random user"
-    }
-
-    CommonAPI.post(Enum.random(users), post)
-  end
-
-  def generate_power_intervals(opts \\ []) do
-    count = Keyword.get(opts, :count, 20)
-    power = Keyword.get(opts, :power, 2)
-    IO.puts("Generating #{count} intervals for a power #{power} series...")
-    counts = Enum.map(1..count, fn n -> :math.pow(n, power) end)
-    sum = Enum.sum(counts)
-
-    densities =
-      Enum.map(counts, fn c ->
-        c / sum
-      end)
-
-    densities
-    |> Enum.reduce(0, fn density, acc ->
-      if acc == 0 do
-        [{0, density}]
-      else
-        [{_, lower} | _] = acc
-        [{lower, lower + density} | acc]
-      end
-    end)
-    |> Enum.reverse()
-  end
-
-  def generate_tagged_activities(opts \\ []) do
-    tag_count = Keyword.get(opts, :tag_count, 20)
-    users = Keyword.get(opts, :users, Repo.all(User))
-    activity_count = Keyword.get(opts, :count, 200_000)
-
-    intervals = generate_power_intervals(count: tag_count)
-
-    IO.puts(
-      "Generating #{activity_count} activities using #{tag_count} different tags of format `tag_n`, starting at tag_0"
-    )
-
-    Enum.each(1..activity_count, fn _ ->
-      random = :rand.uniform()
-      i = Enum.find_index(intervals, fn {lower, upper} -> lower <= random && upper > random end)
-      CommonAPI.post(Enum.random(users), %{"status" => "a post with the tag #tag_#{i}"})
-    end)
-  end
-
-  defp do_generate_activity_with_mention(user, users) do
-    mentions_cnt = Enum.random([2, 3, 4, 5])
-    with_user = Enum.random([true, false])
-    users = Enum.shuffle(users)
-    mentions_users = Enum.take(users, mentions_cnt)
-    mentions_users = if with_user, do: [user | mentions_users], else: mentions_users
-
-    mentions_str =
-      Enum.map(mentions_users, fn user -> "@" <> user.nickname end) |> Enum.join(", ")
-
-    post = %{
-      "status" => mentions_str <> "some status with mentions random users"
-    }
-
-    CommonAPI.post(Enum.random(users), post)
-  end
-
-  defp do_generate_threads(users) do
-    thread_length = Enum.random([2, 3, 4, 5])
-    actor = Enum.random(users)
-
-    post = %{
-      "status" => "Start of the thread"
-    }
-
-    {:ok, activity} = CommonAPI.post(actor, post)
-
-    Enum.each(1..thread_length, fn _ ->
-      user = Enum.random(users)
-
-      post = %{
-        "status" => "@#{actor.nickname} reply to thread",
-        "in_reply_to_status_id" => activity.id
-      }
-
-      CommonAPI.post(user, post)
-    end)
-  end
-
-  def generate_remote_activities(user, users) do
-    do_generate_remote_activities(user, users)
-  end
-
-  defp do_generate_remote_activities(user, users) do
-    IO.puts("Starting generating 10000 remote activities...")
-
-    {time, _} =
-      :timer.tc(fn ->
-        Task.async_stream(
-          1..10_000,
-          fn i ->
-            do_generate_remote_activity(i, user, users)
-          end,
-          max_concurrency: 10,
-          timeout: 30_000
-        )
-        |> Stream.run()
-      end)
-
-    IO.puts("Inserting remote activities take #{to_sec(time)} sec.\n")
-  end
-
-  defp do_generate_remote_activity(i, user, users) do
-    actor = Enum.random(users)
-    %{host: host} = URI.parse(actor.ap_id)
-    date = Date.utc_today()
-    datetime = DateTime.utc_now()
-
-    map = %{
-      "actor" => actor.ap_id,
-      "cc" => [actor.follower_address, user.ap_id],
-      "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
-      "id" => actor.ap_id <> "/statuses/#{i}/activity",
-      "object" => %{
-        "actor" => actor.ap_id,
-        "atomUri" => actor.ap_id <> "/statuses/#{i}",
-        "attachment" => [],
-        "attributedTo" => actor.ap_id,
-        "bcc" => [],
-        "bto" => [],
-        "cc" => [actor.follower_address, user.ap_id],
-        "content" =>
-          "<p><span class=\"h-card\"><a href=\"" <>
-            user.ap_id <>
-            "\" class=\"u-url mention\">@<span>" <> user.nickname <> "</span></a></span></p>",
-        "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
-        "conversation" =>
-          "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
-        "emoji" => %{},
-        "id" => actor.ap_id <> "/statuses/#{i}",
-        "inReplyTo" => nil,
-        "inReplyToAtomUri" => nil,
-        "published" => datetime,
-        "sensitive" => true,
-        "summary" => "cw",
-        "tag" => [
-          %{
-            "href" => user.ap_id,
-            "name" => "@#{user.nickname}@#{host}",
-            "type" => "Mention"
-          }
-        ],
-        "to" => ["https://www.w3.org/ns/activitystreams#Public"],
-        "type" => "Note",
-        "url" => "http://#{host}/@#{actor.nickname}/#{i}"
-      },
-      "published" => datetime,
-      "to" => ["https://www.w3.org/ns/activitystreams#Public"],
-      "type" => "Create"
-    }
-
-    Pleroma.Web.ActivityPub.ActivityPub.insert(map, false)
-  end
-
-  def generate_dms(user, users, opts) do
-    IO.puts("Starting generating #{opts[:dms_max]} DMs")
-    {time, _} = :timer.tc(fn -> do_generate_dms(user, users, opts) end)
-    IO.puts("Inserting dms take #{to_sec(time)} sec.\n")
-  end
-
-  defp do_generate_dms(user, users, opts) do
-    Task.async_stream(
-      1..opts[:dms_max],
-      fn _ ->
-        do_generate_dm(user, users)
-      end,
-      max_concurrency: 10,
-      timeout: 30_000
-    )
-    |> Stream.run()
-  end
-
-  defp do_generate_dm(user, users) do
-    post = %{
-      "status" => "@#{user.nickname} some direct message",
-      "visibility" => "direct"
-    }
-
-    CommonAPI.post(Enum.random(users), post)
-  end
-
-  def generate_long_thread(user, users, opts) do
-    IO.puts("Starting generating long thread with #{opts[:thread_length]} replies")
-    {time, activity} = :timer.tc(fn -> do_generate_long_thread(user, users, opts) end)
-    IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n")
-    {:ok, activity}
-  end
-
-  defp do_generate_long_thread(user, users, opts) do
-    {:ok, %{id: id} = activity} = CommonAPI.post(user, %{"status" => "Start of long thread"})
-
-    Task.async_stream(
-      1..opts[:thread_length],
-      fn _ -> do_generate_thread(users, id) end,
-      max_concurrency: 10,
-      timeout: 30_000
-    )
-    |> Stream.run()
-
-    activity
-  end
-
-  defp do_generate_thread(users, activity_id) do
-    CommonAPI.post(Enum.random(users), %{
-      "status" => "reply to main post",
-      "in_reply_to_status_id" => activity_id
-    })
-  end
-
-  def generate_non_visible_message(user, users) do
-    IO.puts("Starting generating 1000 non visible posts")
-
-    {time, _} =
-      :timer.tc(fn ->
-        do_generate_non_visible_posts(user, users)
-      end)
-
-    IO.puts("Inserting non visible posts take #{to_sec(time)} sec.\n")
-  end
-
-  defp do_generate_non_visible_posts(user, users) do
-    [not_friend | users] = users
-
-    make_friends(user, users)
-
-    Task.async_stream(1..1000, fn _ -> do_generate_non_visible_post(not_friend, users) end,
-      max_concurrency: 10,
-      timeout: 30_000
-    )
-    |> Stream.run()
-  end
-
-  defp make_friends(_user, []), do: nil
-
-  defp make_friends(user, [friend | users]) do
-    {:ok, _} = User.follow(user, friend)
-    {:ok, _} = User.follow(friend, user)
-    make_friends(user, users)
-  end
-
-  defp do_generate_non_visible_post(not_friend, users) do
-    post = %{
-      "status" => "some non visible post",
-      "visibility" => "private"
-    }
-
-    {:ok, activity} = CommonAPI.post(not_friend, post)
-
-    thread_length = Enum.random([2, 3, 4, 5])
-
-    Enum.each(1..thread_length, fn _ ->
-      user = Enum.random(users)
-
-      post = %{
-        "status" => "@#{not_friend.nickname} reply to non visible post",
-        "in_reply_to_status_id" => activity.id,
-        "visibility" => "private"
-      }
-
-      CommonAPI.post(user, post)
-    end)
-  end
-end

benchmarks/load_testing/helper.ex

@@ -1,11 +1,14 @@
 defmodule Pleroma.LoadTesting.Helper do
-  defmacro __using__(_) do
+  alias Ecto.Adapters.SQL
-    quote do
+  alias Pleroma.Repo
-      import Ecto.Query
-      alias Pleroma.Repo
-      alias Pleroma.User
 
-      defp to_sec(microseconds), do: microseconds / 1_000_000
+  def to_sec(microseconds), do: microseconds / 1_000_000
-    end
+
+  def clean_tables do
+    IO.puts("Deleting old data...\n")
+    SQL.query!(Repo, "TRUNCATE users CASCADE;")
+    SQL.query!(Repo, "TRUNCATE activities CASCADE;")
+    SQL.query!(Repo, "TRUNCATE objects CASCADE;")
+    SQL.query!(Repo, "TRUNCATE oban_jobs CASCADE;")
   end
 end

benchmarks/load_testing/users.ex

@@ -0,0 +1,169 @@
+defmodule Pleroma.LoadTesting.Users do
+  @moduledoc """
+  Module for generating users with friends.
+  """
+  import Ecto.Query
+  import Pleroma.LoadTesting.Helper, only: [to_sec: 1]
+
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.User.Query
+
+  @defaults [
+    users: 20_000,
+    friends: 100
+  ]
+
+  @max_concurrency 10
+
+  @spec generate(keyword()) :: User.t()
+  def generate(opts \\ []) do
+    opts = Keyword.merge(@defaults, opts)
+
+    generate_users(opts[:users])
+
+    main_user =
+      Repo.one(from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1))
+
+    make_friends(main_user, opts[:friends])
+
+    Repo.get(User, main_user.id)
+  end
+
+  def generate_users(max) do
+    IO.puts("Starting generating #{max} users...")
+
+    {time, users} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          1..max,
+          &generate_user(&1),
+          max_concurrency: @max_concurrency,
+          timeout: 30_000
+        )
+        |> Enum.to_list()
+      end)
+
+    IO.puts("Generating users took #{to_sec(time)} sec.\n")
+    users
+  end
+
+  defp generate_user(i) do
+    remote = Enum.random([true, false])
+
+    %User{
+      name: "Test テスト User #{i}",
+      email: "user#{i}@example.com",
+      nickname: "nick#{i}",
+      password_hash: Pbkdf2.hash_pwd_salt("test"),
+      bio: "Tester Number #{i}",
+      local: !remote
+    }
+    |> user_urls()
+    |> Repo.insert!()
+  end
+
+  defp user_urls(%{local: true} = user) do
+    urls = %{
+      ap_id: User.ap_id(user),
+      follower_address: User.ap_followers(user),
+      following_address: User.ap_following(user)
+    }
+
+    Map.merge(user, urls)
+  end
+
+  defp user_urls(%{local: false} = user) do
+    base_domain = Enum.random(["domain1.com", "domain2.com", "domain3.com"])
+
+    ap_id = "https://#{base_domain}/users/#{user.nickname}"
+
+    urls = %{
+      ap_id: ap_id,
+      follower_address: ap_id <> "/followers",
+      following_address: ap_id <> "/following"
+    }
+
+    Map.merge(user, urls)
+  end
+
+  def make_friends(main_user, max) when is_integer(max) do
+    IO.puts("Starting making friends for #{max} users...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        number_of_users =
+          (max / 2)
+          |> Kernel.trunc()
+
+        main_user
+        |> get_users(%{limit: number_of_users, local: :local})
+        |> run_stream(main_user)
+
+        main_user
+        |> get_users(%{limit: number_of_users, local: :external})
+        |> run_stream(main_user)
+      end)
+
+    IO.puts("Making friends took #{to_sec(time)} sec.\n")
+  end
+
+  def make_friends(%User{} = main_user, %User{} = user) do
+    {:ok, _} = User.follow(main_user, user)
+    {:ok, _} = User.follow(user, main_user)
+  end
+
+  @spec get_users(User.t(), keyword()) :: [User.t()]
+  def get_users(user, opts) do
+    criteria = %{limit: opts[:limit]}
+
+    criteria =
+      if opts[:local] do
+        Map.put(criteria, opts[:local], true)
+      else
+        criteria
+      end
+
+    criteria =
+      if opts[:friends?] do
+        Map.put(criteria, :friends, user)
+      else
+        criteria
+      end
+
+    query =
+      criteria
+      |> Query.build()
+      |> random_without_user(user)
+
+    query =
+      if opts[:friends?] == false do
+        friends_ids =
+          %{friends: user}
+          |> Query.build()
+          |> Repo.all()
+          |> Enum.map(& &1.id)
+
+        from(u in query, where: u.id not in ^friends_ids)
+      else
+        query
+      end
+
+    Repo.all(query)
+  end
+
+  defp random_without_user(query, user) do
+    from(u in query,
+      where: u.id != ^user.id,
+      order_by: fragment("RANDOM()")
+    )
+  end
+
+  defp run_stream(users, main_user) do
+    Task.async_stream(users, &make_friends(main_user, &1),
+      max_concurrency: @max_concurrency,
+      timeout: 30_000
+    )
+    |> Stream.run()
+  end
+end

benchmarks/mix/tasks/pleroma/benchmarks/tags.ex

@@ -1,9 +1,12 @@
 defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
   use Mix.Task
-  alias Pleroma.Repo
+
-  alias Pleroma.LoadTesting.Generator
+  import Pleroma.LoadTesting.Helper, only: [clean_tables: 0]
   import Ecto.Query
 
+  alias Pleroma.Repo
+  alias Pleroma.Web.MastodonAPI.TimelineController
+
   def run(_args) do
     Mix.Pleroma.start_pleroma()
     activities_count = Repo.aggregate(from(a in Pleroma.Activity), :count, :id)
@@ -11,8 +14,8 @@ def run(_args) do
     if activities_count == 0 do
       IO.puts("Did not find any activities, cleaning and generating")
       clean_tables()
-      Generator.generate_users(users_max: 10)
+      Pleroma.LoadTesting.Users.generate_users(10)
-      Generator.generate_tagged_activities()
+      Pleroma.LoadTesting.Activities.generate_tagged_activities()
     else
       IO.puts("Found #{activities_count} activities, won't generate new ones")
     end
@@ -34,7 +37,7 @@ def run(_args) do
     Benchee.run(
       %{
         "Hashtag fetching, any" => fn tags ->
-          Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
+          TimelineController.hashtag_fetching(
             %{
               "any" => tags
             },
@@ -44,7 +47,7 @@ def run(_args) do
         end,
         # Will always return zero results because no overlapping hashtags are generated.
         "Hashtag fetching, all" => fn tags ->
-          Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
+          TimelineController.hashtag_fetching(
             %{
               "all" => tags
             },
@@ -64,7 +67,7 @@ def run(_args) do
     Benchee.run(
       %{
         "Hashtag fetching" => fn tag ->
-          Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
+          TimelineController.hashtag_fetching(
             %{
               "tag" => tag
             },
@@ -77,11 +80,4 @@ def run(_args) do
       time: 5
     )
   end
-
-  defp clean_tables do
-    IO.puts("Deleting old data...\n")
-    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
-    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
-    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
-  end
 end

benchmarks/mix/tasks/pleroma/benchmarks/timelines.ex

@@ -0,0 +1,70 @@
+defmodule Mix.Tasks.Pleroma.Benchmarks.Timelines do
+  use Mix.Task
+
+  import Pleroma.LoadTesting.Helper, only: [clean_tables: 0]
+
+  alias Pleroma.Web.CommonAPI
+  alias Plug.Conn
+
+  def run(_args) do
+    Mix.Pleroma.start_pleroma()
+
+    # Cleaning tables
+    clean_tables()
+
+    [{:ok, user} | users] = Pleroma.LoadTesting.Users.generate_users(1000)
+
+    # Let the user make 100 posts
+
+    1..100
+    |> Enum.each(fn i -> CommonAPI.post(user, %{"status" => to_string(i)}) end)
+
+    # Let 10 random users post
+    posts =
+      users
+      |> Enum.take_random(10)
+      |> Enum.map(fn {:ok, random_user} ->
+        {:ok, activity} = CommonAPI.post(random_user, %{"status" => "."})
+        activity
+      end)
+
+    # let our user repeat them
+    posts
+    |> Enum.each(fn activity ->
+      CommonAPI.repeat(activity.id, user)
+    end)
+
+    Benchee.run(
+      %{
+        "user timeline, no followers" => fn reading_user ->
+          conn =
+            Phoenix.ConnTest.build_conn()
+            |> Conn.assign(:user, reading_user)
+            |> Conn.assign(:skip_link_headers, true)
+
+          Pleroma.Web.MastodonAPI.AccountController.statuses(conn, %{"id" => user.id})
+        end
+      },
+      inputs: %{"user" => user, "no user" => nil},
+      time: 60
+    )
+
+    users
+    |> Enum.each(fn {:ok, follower} -> Pleroma.User.follow(follower, user) end)
+
+    Benchee.run(
+      %{
+        "user timeline, all following" => fn reading_user ->
+          conn =
+            Phoenix.ConnTest.build_conn()
+            |> Conn.assign(:user, reading_user)
+            |> Conn.assign(:skip_link_headers, true)
+
+          Pleroma.Web.MastodonAPI.AccountController.statuses(conn, %{"id" => user.id})
+        end
+      },
+      inputs: %{"user" => user, "no user" => nil},
+      time: 60
+    )
+  end
+end

benchmarks/mix/tasks/pleroma/load_testing.ex

@@ -1,114 +1,56 @@
 defmodule Mix.Tasks.Pleroma.LoadTesting do
   use Mix.Task
-  use Pleroma.LoadTesting.Helper
+  import Ecto.Query
-  import Mix.Pleroma
+  import Pleroma.LoadTesting.Helper, only: [clean_tables: 0]
-  import Pleroma.LoadTesting.Generator
+
-  import Pleroma.LoadTesting.Fetcher
+  alias Pleroma.Repo
+  alias Pleroma.User
 
   @shortdoc "Factory for generation data"
   @moduledoc """
   Generates data like:
   - local/remote users
-  - local/remote activities with notifications
+  - local/remote activities with differrent visibility:
-  - direct messages
+    - simple activiities
-  - long thread
+    - with emoji
-  - non visible posts
+    - with mentions
+    - hellthreads
+    - with attachments
+    - with tags
+    - likes
+    - reblogs
+    - simple threads
+    - long threads
 
   ## Generate data
-      MIX_ENV=benchmark mix pleroma.load_testing --users 20000 --dms 20000 --thread_length 2000
+      MIX_ENV=benchmark mix pleroma.load_testing --users 20000 --friends 1000 --iterations 170 --friends_used 20 --non_friends_used 20
-      MIX_ENV=benchmark mix pleroma.load_testing -u 20000 -d 20000 -t 2000
+      MIX_ENV=benchmark mix pleroma.load_testing -u 20000 -f 1000 -i 170 -fu 20 -nfu 20
 
   Options:
   - `--users NUMBER` - number of users to generate. Defaults to: 20000. Alias: `-u`
-  - `--dms NUMBER` - number of direct messages to generate. Defaults to: 20000. Alias `-d`
+  - `--friends NUMBER` - number of friends for main user. Defaults to: 1000. Alias: `-f`
-  - `--thread_length` - number of messages in thread. Defaults to: 2000. ALias `-t`
+  - `--iterations NUMBER` - number of iterations to generate activities. For each iteration in database is inserted about 120+ activities with different visibility, actors and types.Defaults to: 170. Alias: `-i`
+  - `--friends_used NUMBER` - number of main user friends used in activity generation. Defaults to: 20. Alias: `-fu`
+  - `--non_friends_used NUMBER` - number of non friends used in activity generation. Defaults to: 20. Alias: `-nfu`
   """
 
-  @aliases [u: :users, d: :dms, t: :thread_length]
+  @aliases [u: :users, f: :friends, i: :iterations, fu: :friends_used, nfu: :non_friends_used]
   @switches [
     users: :integer,
-    dms: :integer,
+    friends: :integer,
-    thread_length: :integer
+    iterations: :integer,
+    friends_used: :integer,
+    non_friends_used: :integer
   ]
-  @users_default 20_000
-  @dms_default 1_000
-  @thread_length_default 2_000
 
   def run(args) do
-    start_pleroma()
+    Logger.configure(level: :error)
-    Pleroma.Config.put([:instance, :skip_thread_containment], true)
+    Mix.Pleroma.start_pleroma()
+    clean_tables()
     {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
 
-    users_max = Keyword.get(opts, :users, @users_default)
+    user = Pleroma.LoadTesting.Users.generate(opts)
-    dms_max = Keyword.get(opts, :dms, @dms_default)
+    Pleroma.LoadTesting.Activities.generate(user, opts)
-    thread_length = Keyword.get(opts, :thread_length, @thread_length_default)
-
-    clean_tables()
-
-    opts =
-      Keyword.put(opts, :users_max, users_max)
-      |> Keyword.put(:dms_max, dms_max)
-      |> Keyword.put(:thread_length, thread_length)
-
-    generate_users(opts)
-
-    # main user for queries
-    IO.puts("Fetching local main user...")
-
-    {time, user} =
-      :timer.tc(fn ->
-        Repo.one(
-          from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1)
-        )
-      end)
-
-    IO.puts("Fetching main user take #{to_sec(time)} sec.\n")
-
-    IO.puts("Fetching local users...")
-
-    {time, users} =
-      :timer.tc(fn ->
-        Repo.all(
-          from(u in User,
-            where: u.id != ^user.id,
-            where: u.local == true,
-            order_by: fragment("RANDOM()"),
-            limit: 10
-          )
-        )
-      end)
-
-    IO.puts("Fetching local users take #{to_sec(time)} sec.\n")
-
-    IO.puts("Fetching remote users...")
-
-    {time, remote_users} =
-      :timer.tc(fn ->
-        Repo.all(
-          from(u in User,
-            where: u.id != ^user.id,
-            where: u.local == false,
-            order_by: fragment("RANDOM()"),
-            limit: 10
-          )
-        )
-      end)
-
-    IO.puts("Fetching remote users take #{to_sec(time)} sec.\n")
-
-    generate_activities(user, users)
-
-    generate_remote_activities(user, remote_users)
-
-    generate_like_activities(
-      user, Pleroma.Repo.all(Pleroma.Activity.Queries.by_type("Create"))
-    )
-
-    generate_dms(user, users, opts)
-
-    {:ok, activity} = generate_long_thread(user, users, opts)
-
-    generate_non_visible_message(user, users)
 
     IO.puts("Users in DB: #{Repo.aggregate(from(u in User), :count, :id)}")
 
@@ -120,19 +62,6 @@ def run(args) do
       "Notifications in DB: #{Repo.aggregate(from(n in Pleroma.Notification), :count, :id)}"
     )
 
-    fetch_user(user)
+    Pleroma.LoadTesting.Fetcher.run_benchmarks(user)
-    query_timelines(user)
-    query_notifications(user)
-    query_dms(user)
-    query_long_thread(user, activity)
-    Pleroma.Config.put([:instance, :skip_thread_containment], false)
-    query_timelines(user)
-  end
-
-  defp clean_tables do
-    IO.puts("Deleting old data...\n")
-    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
-    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
-    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
   end
 end

config/benchmark.exs

@@ -39,7 +39,7 @@
   adapter: Ecto.Adapters.Postgres,
   username: "postgres",
   password: "postgres",
-  database: "pleroma_test",
+  database: "pleroma_benchmark",
   hostname: System.get_env("DB_HOST") || "localhost",
   pool_size: 10
 

config/config.exs

@@ -58,20 +58,6 @@
 
 config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch"
 
-config :pleroma, :hackney_pools,
-  federation: [
-    max_connections: 50,
-    timeout: 150_000
-  ],
-  media: [
-    max_connections: 50,
-    timeout: 150_000
-  ],
-  upload: [
-    max_connections: 25,
-    timeout: 300_000
-  ]
-
 # Upload configuration
 config :pleroma, Pleroma.Upload,
   uploader: Pleroma.Uploaders.Local,
@@ -184,21 +170,13 @@
   "application/ld+json" => ["activity+json"]
 }
 
-config :tesla, adapter: Tesla.Adapter.Hackney
+config :tesla, adapter: Tesla.Adapter.Gun
-
 # Configures http settings, upstream proxy etc.
 config :pleroma, :http,
   proxy_url: nil,
   send_user_agent: true,
   user_agent: :default,
-  adapter: [
+  adapter: []
-    ssl_options: [
-      # Workaround for remote server certificate chain issues
-      partial_chain: &:hackney_connect.partial_chain/1,
-      # We don't support TLS v1.3 yet
-      versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"]
-    ]
-  ]
 
 config :pleroma, :instance,
   name: "Pleroma",
@@ -260,7 +238,20 @@
   account_field_value_length: 2048,
   external_user_synchronization: true,
   extended_nickname_format: true,
-  cleanup_attachments: false
+  cleanup_attachments: false,
+  multi_factor_authentication: [
+    totp: [
+      # digits 6 or 8
+      digits: 6,
+      period: 30
+    ],
+    backup_codes: [
+      number: 5,
+      length: 16
+    ]
+  ]
+
+config :pleroma, :extensions, output_relationships_in_statuses_by_default: true
 
 config :pleroma, :feed,
   post_title: %{
@@ -356,7 +347,8 @@
   reject: [],
   accept: [],
   avatar_removal: [],
-  banner_removal: []
+  banner_removal: [],
+  reject_deletes: []
 
 config :pleroma, :mrf_keyword,
   reject: [],
@@ -624,11 +616,56 @@
   parameters: [gin_fuzzy_search_limit: "500"],
   prepare: :unnamed
 
+config :pleroma, :connections_pool,
+  checkin_timeout: 250,
+  max_connections: 250,
+  retry: 1,
+  retry_timeout: 1000,
+  await_up_timeout: 5_000
+
+config :pleroma, :pools,
+  federation: [
+    size: 50,
+    max_overflow: 10,
+    timeout: 150_000
+  ],
+  media: [
+    size: 50,
+    max_overflow: 10,
+    timeout: 150_000
+  ],
+  upload: [
+    size: 25,
+    max_overflow: 5,
+    timeout: 300_000
+  ],
+  default: [
+    size: 10,
+    max_overflow: 2,
+    timeout: 10_000
+  ]
+
+config :pleroma, :hackney_pools,
+  federation: [
+    max_connections: 50,
+    timeout: 150_000
+  ],
+  media: [
+    max_connections: 50,
+    timeout: 150_000
+  ],
+  upload: [
+    max_connections: 25,
+    timeout: 300_000
+  ]
+
 config :pleroma, :restrict_unauthenticated,
   timelines: %{local: false, federated: false},
   profiles: %{local: false, remote: false},
   activities: %{local: false, remote: false}
 
+config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: false
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

config/description.exs

@@ -28,7 +28,8 @@
       %{
         key: :filters,
         type: {:list, :module},
-        description: "List of filter modules for uploads",
+        description:
+          "List of filter modules for uploads. Module names are shortened (removed leading `Pleroma.Upload.Filter.` part), but on adding custom module you need to use full name.",
         suggestions:
           Generator.list_modules_in_dir(
             "lib/pleroma/upload/filter",
@@ -681,7 +682,8 @@
       %{
         key: :federation_publisher_modules,
         type: {:list, :module},
-        description: "List of modules for federation publishing",
+        description:
+          "List of modules for federation publishing. Module names are shortened (removed leading `Pleroma.Web.` part), but on adding custom module you need to use full name.",
         suggestions: [
           Pleroma.Web.ActivityPub.Publisher
         ]
@@ -694,7 +696,8 @@
       %{
         key: :rewrite_policy,
         type: [:module, {:list, :module}],
-        description: "A list of MRF policies enabled",
+        description:
+          "A list of enabled MRF policies. Module names are shortened (removed leading `Pleroma.Web.ActivityPub.MRF.` part), but on adding custom module you need to use full name.",
         suggestions:
           Generator.list_modules_in_dir(
             "lib/pleroma/web/activity_pub/mrf",
@@ -712,7 +715,7 @@
         key: :quarantined_instances,
         type: {:list, :string},
         description:
-          "List of ActivityPub instances where private (DMs, followers-only) activities will not be send",
+          "List of ActivityPub instances where private (DMs, followers-only) activities will not be sent",
         suggestions: [
           "quarantined.com",
           "*.quarantined.com"
@@ -919,6 +922,62 @@
         key: :external_user_synchronization,
         type: :boolean,
         description: "Enabling following/followers counters synchronization for external users"
+      },
+      %{
+        key: :multi_factor_authentication,
+        type: :keyword,
+        description: "Multi-factor authentication settings",
+        suggestions: [
+          [
+            totp: [digits: 6, period: 30],
+            backup_codes: [number: 5, length: 16]
+          ]
+        ],
+        children: [
+          %{
+            key: :totp,
+            type: :keyword,
+            description: "TOTP settings",
+            suggestions: [digits: 6, period: 30],
+            children: [
+              %{
+                key: :digits,
+                type: :integer,
+                suggestions: [6],
+                description:
+                  "Determines the length of a one-time pass-code, in characters. Defaults to 6 characters."
+              },
+              %{
+                key: :period,
+                type: :integer,
+                suggestions: [30],
+                description:
+                  "a period for which the TOTP code will be valid, in seconds. Defaults to 30 seconds."
+              }
+            ]
+          },
+          %{
+            key: :backup_codes,
+            type: :keyword,
+            description: "MFA backup codes settings",
+            suggestions: [number: 5, length: 16],
+            children: [
+              %{
+                key: :number,
+                type: :integer,
+                suggestions: [5],
+                description: "number of backup codes to generate."
+              },
+              %{
+                key: :length,
+                type: :integer,
+                suggestions: [16],
+                description:
+                  "Determines the length of backup one-time pass-codes, in characters. Defaults to 16 characters."
+              }
+            ]
+          }
+        ]
       }
     ]
   },
@@ -1046,32 +1105,97 @@
         description: "Settings for Pleroma FE",
         suggestions: [
           %{
-            theme: "pleroma-dark",
-            logo: "/static/logo.png",
-            background: "/images/city.jpg",
-            redirectRootNoLogin: "/main/all",
-            redirectRootLogin: "/main/friends",
-            showInstanceSpecificPanel: true,
-            scopeOptionsEnabled: false,
-            formattingOptionsEnabled: false,
-            collapseMessageWithSubject: false,
-            hidePostStats: false,
-            hideUserStats: false,
-            scopeCopy: true,
-            subjectLineBehavior: "email",
             alwaysShowSubjectInput: true,
-            logoMask: false,
+            background: "/static/aurora_borealis.jpg",
+            collapseMessageWithSubject: false,
+            disableChat: false,
+            greentext: false,
+            hideFilteredStatuses: false,
+            hideMutedPosts: false,
+            hidePostStats: false,
+            hideSitename: false,
+            hideUserStats: false,
+            loginMethod: "password",
+            logo: "/static/logo.png",
             logoMargin: ".1em",
-            stickers: false,
+            logoMask: true,
-            enableEmojiPicker: false
+            minimalScopesMode: false,
+            noAttachmentLinks: false,
+            nsfwCensorImage: "",
+            postContentType: "text/plain",
+            redirectRootLogin: "/main/friends",
+            redirectRootNoLogin: "/main/all",
+            scopeCopy: true,
+            showFeaturesPanel: true,
+            showInstanceSpecificPanel: false,
+            subjectLineBehavior: "email",
+            theme: "pleroma-dark",
+            webPushNotifications: false
           }
         ],
         children: [
           %{
-            key: :theme,
+            key: :alwaysShowSubjectInput,
+            label: "Always show subject input",
+            type: :boolean,
+            description: "When disabled, auto-hide the subject field if it's empty"
+          },
+          %{
+            key: :background,
             type: :string,
-            description: "Which theme to use, they are defined in styles.json",
+            description:
-            suggestions: ["pleroma-dark"]
+              "URL of the background, unless viewing a user profile with a background that is set",
+            suggestions: ["/images/city.jpg"]
+          },
+          %{
+            key: :collapseMessageWithSubject,
+            label: "Collapse message with subject",
+            type: :boolean,
+            description:
+              "When a message has a subject (aka Content Warning), collapse it by default"
+          },
+          %{
+            key: :disableChat,
+            label: "PleromaFE Chat",
+            type: :boolean,
+            description: "Disables PleromaFE Chat component"
+          },
+          %{
+            key: :greentext,
+            label: "Greentext",
+            type: :boolean,
+            description: "Enables green text on lines prefixed with the > character."
+          },
+          %{
+            key: :hideFilteredStatuses,
+            label: "Hide Filtered Statuses",
+            type: :boolean,
+            description: "Hides filtered statuses from timelines."
+          },
+          %{
+            key: :hideMutedPosts,
+            label: "Hide Muted Posts",
+            type: :boolean,
+            description: "Hides muted statuses from timelines."
+          },
+          %{
+            key: :hidePostStats,
+            label: "Hide post stats",
+            type: :boolean,
+            description: "Hide notices statistics (repeats, favorites, ...)"
+          },
+          %{
+            key: :hideSitename,
+            label: "Hide Sitename",
+            type: :boolean,
+            description: "Hides instance name from PleromaFE banner."
+          },
+          %{
+            key: :hideUserStats,
+            label: "Hide user stats",
+            type: :boolean,
+            description:
+              "Hide profile statistics (posts, posts per day, followers, followings, ...)"
           },
           %{
             key: :logo,
@@ -1080,11 +1204,44 @@
             suggestions: ["/static/logo.png"]
           },
           %{
-            key: :background,
+            key: :logoMargin,
+            label: "Logo margin",
             type: :string,
             description:
-              "URL of the background, unless viewing a user profile with a background that is set",
+              "Allows you to adjust vertical margins between logo boundary and navbar borders. " <>
-            suggestions: ["/images/city.jpg"]
+                "The idea is that to have logo's image without any extra margins and instead adjust them to your need in layout.",
+            suggestions: [".1em"]
+          },
+          %{
+            key: :logoMask,
+            label: "Logo mask",
+            type: :boolean,
+            description:
+              "By default it assumes logo used will be monochrome with alpha channel to be compatible with both light and dark themes. " <>
+                "If you want a colorful logo you must disable logoMask."
+          },
+          %{
+            key: :minimalScopesMode,
+            label: "Minimal scopes mode",
+            type: :boolean,
+            description:
+              "Limit scope selection to Direct, User default, and Scope of post replying to. " <>
+                "Also prevents replying to a DM with a public post from PleromaFE."
+          },
+          %{
+            key: :nsfwCensorImage,
+            label: "NSFW Censor Image",
+            type: :string,
+            description:
+              "URL of the image to use for hiding NSFW media attachments in the timeline.",
+            suggestions: ["/static/img/nsfw.png"]
+          },
+          %{
+            key: :postContentType,
+            label: "Post Content Type",
+            type: {:dropdown, :atom},
+            description: "Default post formatting option.",
+            suggestions: ["text/plain", "text/html", "text/markdown", "text/bbcode"]
           },
           %{
             key: :redirectRootNoLogin,
@@ -1102,51 +1259,25 @@
               "Relative URL which indicates where to redirect when a user is logged in",
             suggestions: ["/main/friends"]
           },
-          %{
-            key: :showInstanceSpecificPanel,
-            label: "Show instance specific panel",
-            type: :boolean,
-            description: "Whenether to show the instance's specific panel"
-          },
-          %{
-            key: :scopeOptionsEnabled,
-            label: "Scope options enabled",
-            type: :boolean,
-            description: "Enable setting a notice visibility and subject/CW when posting"
-          },
-          %{
-            key: :formattingOptionsEnabled,
-            label: "Formatting options enabled",
-            type: :boolean,
-            description:
-              "Enable setting a formatting different than plain-text (ie. HTML, Markdown) when posting, relates to `:instance`, `allowed_post_formats`"
-          },
-          %{
-            key: :collapseMessageWithSubject,
-            label: "Collapse message with subject",
-            type: :boolean,
-            description:
-              "When a message has a subject (aka Content Warning), collapse it by default"
-          },
-          %{
-            key: :hidePostStats,
-            label: "Hide post stats",
-            type: :boolean,
-            description: "Hide notices statistics (repeats, favorites, ...)"
-          },
-          %{
-            key: :hideUserStats,
-            label: "Hide user stats",
-            type: :boolean,
-            description:
-              "Hide profile statistics (posts, posts per day, followers, followings, ...)"
-          },
           %{
             key: :scopeCopy,
             label: "Scope copy",
             type: :boolean,
             description: "Copy the scope (private/unlisted/public) in replies to posts by default"
           },
+          %{
+            key: :showFeaturesPanel,
+            label: "Show instance features panel",
+            type: :boolean,
+            description:
+              "Enables panel displaying functionality of the instance on the About page."
+          },
+          %{
+            key: :showInstanceSpecificPanel,
+            label: "Show instance specific panel",
+            type: :boolean,
+            description: "Whether to show the instance's custom panel"
+          },
           %{
             key: :subjectLineBehavior,
             label: "Subject line behavior",
@@ -1158,38 +1289,10 @@
             suggestions: ["email", "masto", "noop"]
           },
           %{
-            key: :alwaysShowSubjectInput,
+            key: :theme,
-            label: "Always show subject input",
-            type: :boolean,
-            description: "When disabled, auto-hide the subject field if it's empty"
-          },
-          %{
-            key: :logoMask,
-            label: "Logo mask",
-            type: :boolean,
-            description:
-              "By default it assumes logo used will be monochrome with alpha channel to be compatible with both light and dark themes. " <>
-                "If you want a colorful logo you must disable logoMask."
-          },
-          %{
-            key: :logoMargin,
-            label: "Logo margin",
             type: :string,
-            description:
+            description: "Which theme to use. Available themes are defined in styles.json",
-              "Allows you to adjust vertical margins between logo boundary and navbar borders. " <>
+            suggestions: ["pleroma-dark"]
-                "The idea is that to have logo's image without any extra margins and instead adjust them to your need in layout.",
-            suggestions: [".1em"]
-          },
-          %{
-            key: :stickers,
-            type: :boolean,
-            description: "Enables stickers."
-          },
-          %{
-            key: :enableEmojiPicker,
-            label: "Emoji picker",
-            type: :boolean,
-            description: "Enables emoji picker."
           }
         ]
       },
@@ -1317,13 +1420,13 @@
       %{
         key: :reject,
         type: {:list, :string},
-        description: "List of instances to reject any activities from",
+        description: "List of instances to reject activities from (except deletes)",
         suggestions: ["example.com", "*.example.com"]
       },
       %{
         key: :accept,
         type: {:list, :string},
-        description: "List of instances to accept any activities from",
+        description: "List of instances to only accept activities from (except deletes)",
         suggestions: ["example.com", "*.example.com"]
       },
       %{
@@ -1343,6 +1446,12 @@
         type: {:list, :string},
         description: "List of instances to strip banners from",
         suggestions: ["example.com", "*.example.com"]
+      },
+      %{
+        key: :reject_deletes,
+        type: {:list, :string},
+        description: "List of instances to reject deletions from",
+        suggestions: ["example.com", "*.example.com"]
       }
     ]
   },
@@ -1969,7 +2078,8 @@
       %{
         key: :parsers,
         type: {:list, :module},
-        description: "List of Rich Media parsers.",
+        description:
+          "List of Rich Media parsers. Module names are shortened (removed leading `Pleroma.Web.RichMedia.Parsers.` part), but on adding custom module you need to use full name.",
         suggestions: [
           Pleroma.Web.RichMedia.Parsers.MetaTagsParser,
           Pleroma.Web.RichMedia.Parsers.OEmbed,
@@ -1981,7 +2091,8 @@
         key: :ttl_setters,
         label: "TTL setters",
         type: {:list, :module},
-        description: "List of rich media TTL setters.",
+        description:
+          "List of rich media TTL setters. Module names are shortened (removed leading `Pleroma.Web.RichMedia.Parser.` part), but on adding custom module you need to use full name.",
         suggestions: [
           Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl
         ]
@@ -2241,6 +2352,7 @@
         children: [
           %{
             key: :active,
+            label: "Enabled",
             type: :boolean,
             description: "Globally enable or disable digest emails"
           },
@@ -2442,7 +2554,7 @@
       %{
         key: :relations_actions,
         type: [:tuple, {:list, :tuple}],
-        description: "For actions on relations with all users (follow, unfollow)",
+        description: "For actions on relationships with all users (follow, unfollow)",
         suggestions: [{1000, 10}, [{10_000, 10}, {10_000, 50}]]
       },
       %{
@@ -2654,6 +2766,8 @@
       %{
         key: :scrub_policy,
         type: {:list, :module},
+        description:
+          "Module names are shortened (removed leading `Pleroma.HTML.` part), but on adding custom module you need to use full name.",
         suggestions: [Pleroma.HTML.Transform.MediaProxy, Pleroma.HTML.Scrubber.Default]
       }
     ]
@@ -2916,6 +3030,219 @@
       }
     ]
   },
+  %{
+    group: :pleroma,
+    key: :connections_pool,
+    type: :group,
+    description: "Advanced settings for `gun` connections pool",
+    children: [
+      %{
+        key: :checkin_timeout,
+        type: :integer,
+        description: "Timeout to checkin connection from pool. Default: 250ms.",
+        suggestions: [250]
+      },
+      %{
+        key: :max_connections,
+        type: :integer,
+        description: "Maximum number of connections in the pool. Default: 250 connections.",
+        suggestions: [250]
+      },
+      %{
+        key: :retry,
+        type: :integer,
+        description:
+          "Number of retries, while `gun` will try to reconnect if connection goes down. Default: 1.",
+        suggestions: [1]
+      },
+      %{
+        key: :retry_timeout,
+        type: :integer,
+        description:
+          "Time between retries when `gun` will try to reconnect in milliseconds. Default: 1000ms.",
+        suggestions: [1000]
+      },
+      %{
+        key: :await_up_timeout,
+        type: :integer,
+        description: "Timeout while `gun` will wait until connection is up. Default: 5000ms.",
+        suggestions: [5000]
+      }
+    ]
+  },
+  %{
+    group: :pleroma,
+    key: :pools,
+    type: :group,
+    description: "Advanced settings for `gun` workers pools",
+    children: [
+      %{
+        key: :federation,
+        type: :keyword,
+        description: "Settings for federation pool.",
+        children: [
+          %{
+            key: :size,
+            type: :integer,
+            description: "Number workers in the pool.",
+            suggestions: [50]
+          },
+          %{
+            key: :max_overflow,
+            type: :integer,
+            description: "Number of additional workers if pool is under load.",
+            suggestions: [10]
+          },
+          %{
+            key: :timeout,
+            type: :integer,
+            description: "Timeout while `gun` will wait for response.",
+            suggestions: [150_000]
+          }
+        ]
+      },
+      %{
+        key: :media,
+        type: :keyword,
+        description: "Settings for media pool.",
+        children: [
+          %{
+            key: :size,
+            type: :integer,
+            description: "Number workers in the pool.",
+            suggestions: [50]
+          },
+          %{
+            key: :max_overflow,
+            type: :integer,
+            description: "Number of additional workers if pool is under load.",
+            suggestions: [10]
+          },
+          %{
+            key: :timeout,
+            type: :integer,
+            description: "Timeout while `gun` will wait for response.",
+            suggestions: [150_000]
+          }
+        ]
+      },
+      %{
+        key: :upload,
+        type: :keyword,
+        description: "Settings for upload pool.",
+        children: [
+          %{
+            key: :size,
+            type: :integer,
+            description: "Number workers in the pool.",
+            suggestions: [25]
+          },
+          %{
+            key: :max_overflow,
+            type: :integer,
+            description: "Number of additional workers if pool is under load.",
+            suggestions: [5]
+          },
+          %{
+            key: :timeout,
+            type: :integer,
+            description: "Timeout while `gun` will wait for response.",
+            suggestions: [300_000]
+          }
+        ]
+      },
+      %{
+        key: :default,
+        type: :keyword,
+        description: "Settings for default pool.",
+        children: [
+          %{
+            key: :size,
+            type: :integer,
+            description: "Number workers in the pool.",
+            suggestions: [10]
+          },
+          %{
+            key: :max_overflow,
+            type: :integer,
+            description: "Number of additional workers if pool is under load.",
+            suggestions: [2]
+          },
+          %{
+            key: :timeout,
+            type: :integer,
+            description: "Timeout while `gun` will wait for response.",
+            suggestions: [10_000]
+          }
+        ]
+      }
+    ]
+  },
+  %{
+    group: :pleroma,
+    key: :hackney_pools,
+    type: :group,
+    description: "Advanced settings for `hackney` connections pools",
+    children: [
+      %{
+        key: :federation,
+        type: :keyword,
+        description: "Settings for federation pool.",
+        children: [
+          %{
+            key: :max_connections,
+            type: :integer,
+            description: "Number workers in the pool.",
+            suggestions: [50]
+          },
+          %{
+            key: :timeout,
+            type: :integer,
+            description: "Timeout while `hackney` will wait for response.",
+            suggestions: [150_000]
+          }
+        ]
+      },
+      %{
+        key: :media,
+        type: :keyword,
+        description: "Settings for media pool.",
+        children: [
+          %{
+            key: :max_connections,
+            type: :integer,
+            description: "Number workers in the pool.",
+            suggestions: [50]
+          },
+          %{
+            key: :timeout,
+            type: :integer,
+            description: "Timeout while `hackney` will wait for response.",
+            suggestions: [150_000]
+          }
+        ]
+      },
+      %{
+        key: :upload,
+        type: :keyword,
+        description: "Settings for upload pool.",
+        children: [
+          %{
+            key: :max_connections,
+            type: :integer,
+            description: "Number workers in the pool.",
+            suggestions: [25]
+          },
+          %{
+            key: :timeout,
+            type: :integer,
+            description: "Timeout while `hackney` will wait for response.",
+            suggestions: [300_000]
+          }
+        ]
+      }
+    ]
+  },
   %{
     group: :pleroma,
     key: :restrict_unauthenticated,
@@ -2975,5 +3302,19 @@
         ]
       }
     ]
+  },
+  %{
+    group: :pleroma,
+    key: Pleroma.Web.ApiSpec.CastAndValidate,
+    type: :group,
+    children: [
+      %{
+        key: :strict,
+        type: :boolean,
+        description:
+          "Enables strict input validation (useful in development, not recommended in production)",
+        suggestions: [false]
+      }
+    ]
   }
 ]

config/dev.exs

@@ -52,6 +52,8 @@
   hostname: "localhost",
   pool_size: 10
 
+config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: true
+
 if File.exists?("./config/dev.secret.exs") do
   import_config "dev.secret.exs"
 else

config/test.exs

@@ -56,6 +56,19 @@
   ignore_hosts: [],
   ignore_tld: ["local", "localdomain", "lan"]
 
+config :pleroma, :instance,
+  multi_factor_authentication: [
+    totp: [
+      # digits 6 or 8
+      digits: 6,
+      period: 30
+    ],
+    backup_codes: [
+      number: 2,
+      length: 6
+    ]
+  ]
+
 config :web_push_encryption, :vapid_details,
   subject: "mailto:administrator@example.com",
   public_key:
@@ -90,10 +103,14 @@
 
 config :pleroma, :modules, runtime_dir: "test/fixtures/modules"
 
+config :pleroma, Pleroma.Gun, Pleroma.GunMock
+
 config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: true
 
 config :pleroma, Pleroma.Plugs.RemoteIp, enabled: false
 
+config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: true
+
 if File.exists?("./config/test.secret.exs") do
   import_config "test.secret.exs"
 else

coveralls.json

@@ -0,0 +1,6 @@
+{
+  "skip_files": [
+    "test/support",
+    "lib/mix/tasks/pleroma/benchmark.ex"
+  ]
+}

docs/API/admin_api.md

@@ -392,10 +392,24 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   - `email`
   - `name`, optional
 
+- Response:
+  - On success: `204`, empty response
+  - On failure:
+    - 400 Bad Request, JSON:
+
+    ```json
+      [
+        {
+          "error": "Appropriate error message here"
+        }
+      ]
+    ```
+
 ## `GET /api/pleroma/admin/users/:nickname/password_reset`
 
 ### Get a password reset token for a given nickname
 
+
 - Params: none
 - Response:
 
@@ -414,6 +428,91 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   - `nicknames`
 - Response: none (code `204`)
 
+## PUT `/api/pleroma/admin/users/disable_mfa`
+
+### Disable mfa for user's account.
+
+- Params:
+  - `nickname`
+- Response: User’s nickname
+
+## `GET /api/pleroma/admin/users/:nickname/credentials`
+
+### Get the user's email, password, display and settings-related fields
+
+- Params:
+  - `nickname`
+
+- Response:
+
+```json
+{
+  "actor_type": "Person",
+  "allow_following_move": true,
+  "avatar": "https://pleroma.social/media/7e8e7508fd545ef580549b6881d80ec0ff2c81ed9ad37b9bdbbdf0e0d030159d.jpg",
+  "background": "https://pleroma.social/media/4de34c0bd10970d02cbdef8972bef0ebbf55f43cadc449554d4396156162fe9a.jpg",
+  "banner": "https://pleroma.social/media/8d92ba2bd244b613520abf557dd448adcd30f5587022813ee9dd068945986946.jpg",
+  "bio": "bio",
+  "default_scope": "public",
+  "discoverable": false,
+  "email": "user@example.com",
+  "fields": [
+    {
+      "name": "example",
+      "value": "<a href=\"https://example.com\" rel=\"ugc\">https://example.com</a>"
+    }
+  ],
+  "hide_favorites": false,
+  "hide_followers": false,
+  "hide_followers_count": false,
+  "hide_follows": false,
+  "hide_follows_count": false,
+  "id": "9oouHaEEUR54hls968",
+  "locked": true,
+  "name": "user",
+  "no_rich_text": true,
+  "pleroma_settings_store": {},
+  "raw_fields": [
+    {
+      "id": 1,
+      "name": "example",
+      "value": "https://example.com"
+    },
+  ],
+  "show_role": true,
+  "skip_thread_containment": false
+}
+```
+
+## `PATCH /api/pleroma/admin/users/:nickname/credentials`
+
+### Change the user's email, password, display and settings-related fields
+
+- Params:
+  - `email`
+  - `password`
+  - `name`
+  - `bio`
+  - `avatar`
+  - `locked`
+  - `no_rich_text`
+  - `default_scope`
+  - `banner`
+  - `hide_follows`
+  - `hide_followers`
+  - `hide_followers_count`
+  - `hide_follows_count`
+  - `hide_favorites`
+  - `allow_following_move`
+  - `background`
+  - `show_role`
+  - `skip_thread_containment`
+  - `fields`
+  - `discoverable`
+  - `actor_type`
+
+- Response: none (code `200`)
+
 ## `GET /api/pleroma/admin/reports`
 
 ### Get a list of reports
@@ -665,6 +764,17 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
     - 400 Bad Request `"Invalid parameters"` when `status` is missing
   - On success: `204`, empty response
 
+## `GET /api/pleroma/admin/statuses/:id`
+
+### Show status by id
+
+- Params:
+  - `id`: required, status id
+- Response:
+  - On failure:
+    - 404 Not Found `"Not Found"`
+  - On success: JSON, Mastodon Status entity
+
 ## `PUT /api/pleroma/admin/statuses/:id`
 
 ### Change the scope of an individual reported status
@@ -696,6 +806,8 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 
 ### Restarts pleroma application
 
+**Only works when configuration from database is enabled.**
+
 - Params: none
 - Response:
   - On failure:
@@ -705,11 +817,24 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 {}
 ```
 
+## `GET /api/pleroma/admin/need_reboot`
+
+### Returns the flag whether the pleroma should be restarted
+
+- Params: none
+- Response:
+  - `need_reboot` - boolean
+```json
+{
+  "need_reboot": false
+}
+```
+
 ## `GET /api/pleroma/admin/config`
 
 ### Get list of merged default settings with saved in database.
 
-*If `need_reboot` flag exists in response, instance must be restarted, so reboot time settings can take effect.*
+*If `need_reboot` is `true`, instance must be restarted, so reboot time settings can take effect.*
 
 **Only works when configuration from database is enabled.**
 
@@ -731,13 +856,12 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   "need_reboot": true
 }
 ```
- need_reboot - *optional*, if were changed reboot time settings.
 
 ## `POST /api/pleroma/admin/config`
 
 ### Update config settings
 
-*If `need_reboot` flag exists in response, instance must be restarted, so reboot time settings can take effect.*
+*If `need_reboot` is `true`, instance must be restarted, so reboot time settings can take effect.*
 
 **Only works when configuration from database is enabled.**
 
@@ -764,6 +888,8 @@ Some modifications are necessary to save the config settings correctly:
 Most of the settings will be applied in `runtime`, this means that you don't need to restart the instance. But some settings are applied in `compile time` and require a reboot of the instance, such as:
 - all settings inside these keys:
   - `:hackney_pools`
+  - `:connections_pool`
+  - `:pools`
   - `:chat`
 - partially settings inside these keys:
   - `:seconds_valid` in `Pleroma.Captcha`
@@ -879,7 +1005,6 @@ config :quack,
   "need_reboot": true
 }
 ```
-need_reboot - *optional*, if were changed reboot time settings.
 
 ## ` GET /api/pleroma/admin/config/descriptions`
 
@@ -983,3 +1108,104 @@ Loads json generated from `config/descriptions.exs`.
   }
 }
 ```
+
+## `GET /api/pleroma/admin/oauth_app`
+
+### List OAuth app
+
+- Params:
+  - *optional* `name`
+  - *optional* `client_id`
+  - *optional* `page`
+  - *optional* `page_size`
+  - *optional* `trusted`
+
+- Response:
+
+```json
+{
+  "apps": [
+    {
+      "id": 1,
+      "name": "App name",
+      "client_id": "yHoDSiWYp5mPV6AfsaVOWjdOyt5PhWRiafi6MRd1lSk",
+      "client_secret": "nLmis486Vqrv2o65eM9mLQx_m_4gH-Q6PcDpGIMl6FY",
+      "redirect_uri": "https://example.com/oauth-callback",
+      "website": "https://example.com",
+      "trusted": true
+    }
+  ],
+  "count": 17,
+  "page_size": 50
+}
+```
+
+
+## `POST /api/pleroma/admin/oauth_app`
+
+### Create OAuth App
+
+- Params:
+  - `name`
+  - `redirect_uris`
+  - `scopes`
+  - *optional* `website`
+  - *optional* `trusted`
+
+- Response:
+
+```json
+{
+  "id": 1,
+  "name": "App name",
+  "client_id": "yHoDSiWYp5mPV6AfsaVOWjdOyt5PhWRiafi6MRd1lSk",
+  "client_secret": "nLmis486Vqrv2o65eM9mLQx_m_4gH-Q6PcDpGIMl6FY",
+  "redirect_uri": "https://example.com/oauth-callback",
+  "website": "https://example.com",
+  "trusted": true
+}
+```
+
+- On failure:
+```json
+{
+  "redirect_uris": "can't be blank",
+  "name": "can't be blank"
+}
+```
+
+## `PATCH /api/pleroma/admin/oauth_app/:id`
+
+### Update OAuth App
+
+- Params:
+  -  *optional* `name`
+  -  *optional* `redirect_uris`
+  -  *optional* `scopes`
+  -  *optional* `website`
+  -  *optional* `trusted`
+
+- Response:
+
+```json
+{
+  "id": 1,
+  "name": "App name",
+  "client_id": "yHoDSiWYp5mPV6AfsaVOWjdOyt5PhWRiafi6MRd1lSk",
+  "client_secret": "nLmis486Vqrv2o65eM9mLQx_m_4gH-Q6PcDpGIMl6FY",
+  "redirect_uri": "https://example.com/oauth-callback",
+  "website": "https://example.com",
+  "trusted": true
+}
+```
+
+## `DELETE /api/pleroma/admin/oauth_app/:id`
+
+### Delete OAuth App
+
+- Params: None
+
+- Response:
+  - On success: `204`, empty response
+  - On failure:
+    - 400 Bad Request `"Invalid parameters"` when `status` is missing

docs/API/differences_in_mastoapi_responses.md

@@ -4,7 +4,7 @@ A Pleroma instance can be identified by "<Mastodon version> (compatible; Pleroma
 
 ## Flake IDs
 
-Pleroma uses 128-bit ids as opposed to Mastodon's 64 bits. However just like Mastodon's ids they are sortable strings
+Pleroma uses 128-bit ids as opposed to Mastodon's 64 bits. However just like Mastodon's ids they are lexically sortable strings
 
 ## Attachment cap
 
@@ -14,6 +14,7 @@ Some apps operate under the assumption that no more than 4 attachments can be re
 
 Adding the parameter `with_muted=true` to the timeline queries will also return activities by muted (not by blocked!) users.
 Adding the parameter `exclude_visibilities` to the timeline queries will exclude the statuses with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`), e.g., `exclude_visibilities[]=direct&exclude_visibilities[]=private`.
+Adding the parameter `reply_visibility` to the public and home timelines queries will filter replies. Possible values: without parameter (default) shows all replies, `following` - replies directed to you or users you follow, `self` - replies directed to you.
 
 ## Statuses
 
@@ -60,6 +61,7 @@ Has these additional fields under the `pleroma` object:
 - `deactivated`: boolean, true when the user is deactivated
 - `allow_following_move`: boolean, true when the user allows automatically follow moved following accounts
 - `unread_conversation_count`: The count of unread conversations. Only returned to the account owner.
+- `unread_notifications_count`: The count of unread notifications. Only returned to the account owner.
 
 ### Source
 
@@ -119,6 +121,18 @@ Accepts additional parameters:
 - `exclude_visibilities`: will exclude the notifications for activities with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`). Usage example: `GET /api/v1/notifications?exclude_visibilities[]=direct&exclude_visibilities[]=private`.
 - `include_types`: will include the notifications for activities with the given types. The parameter accepts an array of types (`mention`, `follow`, `reblog`, `favourite`, `move`, `pleroma:emoji_reaction`). Usage example: `GET /api/v1/notifications?include_types[]=mention&include_types[]=reblog`.
 
+## DELETE `/api/v1/notifications/destroy_multiple`
+
+An endpoint to delete multiple statuses by IDs.
+
+Required parameters:
+
+- `ids`: array of activity ids
+
+Usage example: `DELETE /api/v1/notifications/destroy_multiple/?ids[]=1&ids[]=2`.
+
+Returns on success: 200 OK `{}`
+
 ## POST `/api/v1/statuses`
 
 Additional parameters can be added to the JSON body/Form data:
@@ -164,6 +178,7 @@ Additional parameters can be added to the JSON body/Form data:
 - `actor_type` - the type of this account.
 
 ### Pleroma Settings Store
+
 Pleroma has mechanism that allows frontends to save blobs of json for each user on the backend. This can be used to save frontend-specific settings for a user that the backend does not need to know about.
 
 The parameter should have a form of `{frontend_name: {...}}`, with `frontend_name` identifying your type of client, e.g. `pleroma_fe`. It will overwrite everything under this property, but will not overwrite other frontend's settings.
@@ -172,17 +187,41 @@ This information is returned in the `verify_credentials` endpoint.
 
 ## Authentication
 
-*Pleroma supports refreshing tokens.
+*Pleroma supports refreshing tokens.*
 
 `POST /oauth/token`
-Post here request with grant_type=refresh_token to obtain new access token. Returns an access token.
+
+Post here request with `grant_type=refresh_token` to obtain new access token. Returns an access token.
 
 ## Account Registration
+
 `POST /api/v1/accounts`
 
 Has theses additional parameters (which are the same as in Pleroma-API):
-    * `fullname`: optional
+
-    * `bio`: optional
+- `fullname`: optional
-    * `captcha_solution`: optional, contains provider-specific captcha solution,
+- `bio`: optional
-    * `captcha_token`: optional, contains provider-specific captcha token
+- `captcha_solution`: optional, contains provider-specific captcha solution,
-    * `token`: invite token required when the registerations aren't public.
+- `captcha_token`: optional, contains provider-specific captcha token
+- `captcha_answer_data`: optional, contains provider-specific captcha data
+- `token`: invite token required when the registrations aren't public.
+
+## Instance
+
+`GET /api/v1/instance` has additional fields
+
+- `max_toot_chars`: The maximum characters per post
+- `poll_limits`: The limits of polls
+- `upload_limit`: The maximum upload file size
+- `avatar_upload_limit`: The same for avatars
+- `background_upload_limit`: The same for backgrounds
+- `banner_upload_limit`: The same for banners
+- `pleroma.metadata.features`: A list of supported features
+- `pleroma.metadata.federation`: The federation restrictions of this instance
+- `vapid_public_key`: The public key needed for push messages
+
+## Markers
+
+Has these additional fields under the `pleroma` object:
+
+- `unread_count`: contains number unread notifications

docs/API/pleroma_api.md

@@ -70,7 +70,49 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 * Response: JSON. Returns `{"status": "success"}` if the account was successfully disabled, `{"error": "[error message]"}` otherwise
 * Example response: `{"error": "Invalid password."}`
 
-## `/api/pleroma/admin/`…
+## `/api/pleroma/accounts/mfa`
+#### Gets current MFA settings
+* method: `GET`
+* Authentication: required
+* OAuth scope: `read:security`
+* Response: JSON. Returns `{"enabled": "false", "totp": false }`
+
+## `/api/pleroma/accounts/mfa/setup/totp`
+#### Pre-setup the MFA/TOTP method
+* method: `GET`
+* Authentication: required
+* OAuth scope: `write:security`
+* Response: JSON. Returns `{"key": [secret_key], "provisioning_uri": "[qr code uri]"  }` when successful, otherwise returns HTTP 422 `{"error": "error_msg"}`
+
+## `/api/pleroma/accounts/mfa/confirm/totp`
+#### Confirms & enables MFA/TOTP support for user account.
+* method: `POST`
+* Authentication: required
+* OAuth scope: `write:security`
+* Params:
+    * `password`: user's password
+    * `code`: token from TOTP App
+* Response: JSON. Returns `{}` if the enable was successful, HTTP 422 `{"error": "[error message]"}` otherwise
+
+
+## `/api/pleroma/accounts/mfa/totp`
+####  Disables MFA/TOTP method for user account.
+* method: `DELETE`
+* Authentication: required
+* OAuth scope: `write:security`
+* Params:
+    * `password`: user's password
+* Response: JSON. Returns `{}` if the disable was successful, HTTP 422 `{"error": "[error message]"}` otherwise
+* Example response: `{"error": "Invalid password."}`
+
+## `/api/pleroma/accounts/mfa/backup_codes`
+####  Generstes backup codes MFA for user account.
+* method: `GET`
+* Authentication: required
+* OAuth scope: `write:security`
+* Response: JSON. Returns `{"codes": codes}`when successful, otherwise HTTP 422 `{"error": "[error message]"}`
+
+## `/api/pleroma/admin/`
 See [Admin-API](admin_api.md)
 
 ## `/api/v1/pleroma/notifications/read`
@@ -323,20 +365,54 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 * Params: None
 * Response: JSON, returns a list of Mastodon Conversation entities that were marked as read (200 - healthy, 503 unhealthy).
 
-## `GET /api/pleroma/emoji/packs`
+## `GET /api/pleroma/emoji/packs/import`
-### Lists the custom emoji packs on the server
+### Imports packs from filesystem
 * Method `GET`
-* Authentication: not required
+* Authentication: required
 * Params: None
-* Response: JSON, "ok" and 200 status and the JSON hashmap of "pack name" to "pack contents"
+* Response: JSON, returns a list of imported packs.
 
-## `PUT /api/pleroma/emoji/packs/:name`
+## `GET /api/pleroma/emoji/packs/remote`
-### Creates an empty custom emoji pack
+### Make request to another instance for packs list
-* Method `PUT`
+* Method `GET`
+* Authentication: required
+* Params:
+  * `url`: url of the instance to get packs from
+* Response: JSON with the pack list, hashmap with pack name and pack contents
+
+## `POST /api/pleroma/emoji/packs/download`
+### Download pack from another instance
+* Method `POST`
+* Authentication: required
+* Params:
+  * `url`: url of the instance to download from
+  * `name`: pack to download from that instance
+  * `as`: (*optional*) name how to save pack
+* Response: JSON, "ok" with 200 status if the pack was downloaded, or 500 if there were
+  errors downloading the pack
+
+## `POST /api/pleroma/emoji/packs/:name`
+### Creates an empty pack
+* Method `POST`
 * Authentication: required
 * Params: None
 * Response: JSON, "ok" and 200 status or 409 if the pack with that name already exists
 
+## `PATCH /api/pleroma/emoji/packs/:name`
+### Updates (replaces) pack metadata
+* Method `PATCH`
+* Authentication: required
+* Params:
+  * `metadata`: metadata to replace the old one
+    * `license`: Pack license
+    * `homepage`: Pack home page url
+    * `description`: Pack description
+    * `fallback-src`: Fallback url to download pack from
+    * `fallback-src-sha256`: SHA256 encoded for fallback pack archive
+    * `share-files`: is pack allowed for sharing (boolean)
+* Response: JSON, updated "metadata" section of the pack and 200 status or 400 if there was a
+  problem with the new metadata (the error is specified in the "error" part of the response JSON)
+
 ## `DELETE /api/pleroma/emoji/packs/:name`
 ### Delete a custom emoji pack
 * Method `DELETE`
@@ -344,53 +420,51 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 * Params: None
 * Response: JSON, "ok" and 200 status or 500 if there was an error deleting the pack
 
-## `POST /api/pleroma/emoji/packs/:name/update_file`
+## `POST /api/pleroma/emoji/packs/:name/files`
-### Update a file in a custom emoji pack
+### Add new file to the pack
 * Method `POST`
 * Authentication: required
 * Params:
-    * if the `action` is `add`, adds an emoji named `shortcode` to the pack `pack_name`,
+  * `file`: file needs to be uploaded with the multipart request or link to remote file.
-      that means that the emoji file needs to be uploaded with the request
+  * `shortcode`: (*optional*) shortcode for new emoji, must be uniq for all emoji. If not sended, shortcode will be taken from original filename.
-      (thus requiring it to be a multipart request) and be named `file`.
+  * `filename`: (*optional*) new emoji file name. If not specified will be taken from original filename.
-      There can also be an optional `filename` that will be the new emoji file name
+* Response: JSON, list of files for updated pack (hashmap -> shortcode => filename) with status 200, either error status with error message.
-      (if it's not there, the name will be taken from the uploaded file).
-    * if the `action` is `update`, changes emoji shortcode
-      (from `shortcode` to `new_shortcode` or moves the file (from the current filename to `new_filename`)
-    * if the `action` is `remove`, removes the emoji named `shortcode` and it's associated file
-* Response: JSON, updated "files" section of the pack and 200 status, 409 if the trying to use a shortcode
-  that is already taken, 400 if there was an error with the shortcode, filename or file (additional info
-  in the "error" part of the response JSON)
 
-## `POST /api/pleroma/emoji/packs/:name/update_metadata`
+## `PATCH /api/pleroma/emoji/packs/:name/files`
-### Updates (replaces) pack metadata
+### Update emoji file from pack
-* Method `POST`
+* Method `PATCH`
 * Authentication: required
 * Params:
-  * `new_data`: new metadata to replace the old one
+  * `shortcode`: emoji file shortcode
-* Response: JSON, updated "metadata" section of the pack and 200 status or 400 if there was a
+  * `new_shortcode`: new emoji file shortcode
-  problem with the new metadata (the error is specified in the "error" part of the response JSON)
+  * `new_filename`: new filename for emoji file
+  * `force`: (*optional*) with true value to overwrite existing emoji with new shortcode
+* Response: JSON, list with updated files for updated pack (hashmap -> shortcode => filename) with status 200, either error status with error message.
 
-## `POST /api/pleroma/emoji/packs/download_from`
+## `DELETE /api/pleroma/emoji/packs/:name/files`
-### Requests the instance to download the pack from another instance
+### Delete emoji file from pack
-* Method `POST`
+* Method `DELETE`
 * Authentication: required
 * Params:
-  * `instance_address`: the address of the instance to download from
+  * `shortcode`: emoji file shortcode
-  * `pack_name`: the pack to download from that instance
+* Response: JSON, list with updated files for updated pack (hashmap -> shortcode => filename) with status 200, either error status with error message.
-* Response: JSON, "ok" and 200 status if the pack was downloaded, or 500 if there were
-  errors downloading the pack
 
-## `POST /api/pleroma/emoji/packs/list_from`
+## `GET /api/pleroma/emoji/packs`
-### Requests the instance to list the packs from another instance
+### Lists local custom emoji packs
-* Method `POST`
+* Method `GET`
-* Authentication: required
+* Authentication: not required
-* Params:
+* Params: None
-  * `instance_address`: the address of the instance to download from
+* Response: JSON, "ok" and 200 status and the JSON hashmap of pack name to pack contents
-* Response: JSON with the pack list, same as if the request was made to that instance's
-  list endpoint directly + 200 status
 
-## `GET /api/pleroma/emoji/packs/:name/download_shared`
+## `GET /api/pleroma/emoji/packs/:name`
-### Requests a local pack from the instance
+### Get pack.json for the pack
+* Method `GET`
+* Authentication: not required
+* Params: None
+* Response: JSON, pack json with `files` and `pack` keys with 200 status or 404 if the pack does not exist
+
+## `GET /api/pleroma/emoji/packs/:name/archive`
+### Requests a local pack archive from the instance
 * Method `GET`
 * Authentication: not required
 * Params: None
@@ -431,7 +505,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 
 # Emoji Reactions
 
-Emoji reactions work a lot like favourites do. They make it possible to react to a post with a single emoji character.
+Emoji reactions work a lot like favourites do. They make it possible to react to a post with a single emoji character. To detect the presence of this feature, you can check `pleroma_emoji_reactions` entry in the features list of nodeinfo.
 
 ## `PUT /api/v1/pleroma/statuses/:id/reactions/:emoji`
 ### React to a post with a unicode emoji

docs/administration/CLI_tasks/emoji.md

@@ -39,8 +39,8 @@ mix pleroma.emoji get-packs [option ...] <pack ...>
 mix pleroma.emoji gen-pack PACK-URL
 ```
 
-Currently, only .zip archives are recognized as remote pack files and packs are therefore assumed to be zip archives. This command is intended to run interactively and will first ask you some basic questions about the pack, then download the remote file and generate an SHA256 checksum for it, then generate an emoji file list for you. 
+Currently, only .zip archives are recognized as remote pack files and packs are therefore assumed to be zip archives. This command is intended to run interactively and will first ask you some basic questions about the pack, then download the remote file and generate an SHA256 checksum for it, then generate an emoji file list for you.
 
-  The manifest entry will either be written to a newly created `index.json` file or appended to the existing one, *replacing* the old pack with the same name if it was in the file previously.
+  The manifest entry will either be written to a newly created `pack_name.json` file (pack name is asked in questions) or appended to the existing one, *replacing* the old pack with the same name if it was in the file previously.
 
   The file list will be written to the file specified previously, *replacing* that file. You _should_ check that the file list doesn't contain anything you don't need in the pack, that is, anything that is not an emoji (the whole pack is downloaded, but only emoji files are extracted).

docs/administration/CLI_tasks/oauth_app.md

@@ -0,0 +1,16 @@
+# Creating trusted OAuth App
+
+{! backend/administration/CLI_tasks/general_cli_task_info.include !}
+
+## Create trusted OAuth App.
+
+Optional params:
+  * `-s SCOPES` - scopes for app, e.g. `read,write,follow,push`.
+
+```sh tab="OTP"
+ ./bin/pleroma_ctl app create -n APP_NAME -r REDIRECT_URI
+```
+
+```sh tab="From Source"
+mix pleroma.app create -n APP_NAME -r REDIRECT_URI
+```

docs/clients.md

@@ -49,11 +49,11 @@ Feel free to contact us to be added to this list!
 - Platforms: Android
 - Features: Streaming Ready
 
-### Roma
+### Fedi
-- Homepage: <https://www.pleroma.com/#mobileApps>
+- Homepage: <https://www.fediapp.com/>
-- Source Code: [iOS](https://github.com/roma-apps/roma-ios), [Android](https://github.com/roma-apps/roma-android)
+- Source Code: Proprietary, but free
 - Platforms: iOS, Android
-- Features: No Streaming
+- Features: Pleroma-specific features like Reactions
 
 ### Tusky
 - Homepage: <https://tuskyapp.github.io/>

docs/configuration/cheatsheet.md

@@ -8,6 +8,10 @@ For from source installations Pleroma configuration works by first importing the
 
 To add configuration to your config file, you can copy it from the base config. The latest version of it can be viewed [here](https://git.pleroma.social/pleroma/pleroma/blob/develop/config/config.exs). You can also use this file if you don't know how an option is supposed to be formatted.
 
+## :chat
+
+* `enabled` - Enables the backend chat. Defaults to `true`.
+
 ## :instance
 * `name`: The instance’s name.
 * `email`: Email used to reach an Administrator/Moderator of the instance.
@@ -369,8 +373,7 @@ Available caches:
 * `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`)
 * `send_user_agent`: should we include a user agent with HTTP requests? (default: `true`)
 * `user_agent`: what user agent should we use? (default: `:default`), must be string or `:default`
-* `adapter`: array of hackney options
+* `adapter`: array of adapter options
-
 
 ### :hackney_pools
 
@@ -389,6 +392,42 @@ For each pool, the options are:
 * `timeout` - retention duration for connections
 
 
+### :connections_pool
+
+*For `gun` adapter*
+
+Advanced settings for connections pool. Pool with opened connections. These connections can be reused in worker pools.
+
+For big instances it's recommended to increase `config :pleroma, :connections_pool, max_connections: 500` up to 500-1000.
+It will increase memory usage, but federation would work faster.
+
+* `:checkin_timeout` - timeout to checkin connection from pool. Default: 250ms.
+* `:max_connections` - maximum number of connections in the pool. Default: 250 connections.
+* `:retry` - number of retries, while `gun` will try to reconnect if connection goes down. Default: 1.
+* `:retry_timeout` - time between retries when `gun` will try to reconnect in milliseconds. Default: 1000ms.
+* `:await_up_timeout` - timeout while `gun` will wait until connection is up. Default: 5000ms.
+
+### :pools
+
+*For `gun` adapter*
+
+Advanced settings for workers pools.
+
+There are four pools used:
+
+* `:federation` for the federation jobs.
+  You may want this pool max_connections to be at least equal to the number of federator jobs + retry queue jobs.
+* `:media` for rich media, media proxy
+* `:upload` for uploaded media (if using a remote uploader and `proxy_remote: true`)
+* `:default` for other requests
+
+For each pool, the options are:
+
+* `:size` - how much workers the pool can hold
+* `:timeout` - timeout while `gun` will wait for response
+* `:max_overflow` - additional workers if pool is under load
+
+
 ## Captcha
 
 ### Pleroma.Captcha
@@ -868,12 +907,33 @@ config :auto_linker,
 
 * `runtime_dir`: A path to custom Elixir modules (such as MRF policies).
 
-
 ## :configurable_from_database
 
 Boolean, enables/disables in-database configuration. Read [Transfering the config to/from the database](../administration/CLI_tasks/config.md) for more information.
 
+## :database_config_whitelist
 
+List of valid configuration sections which are allowed to be configured from the
+database. Settings stored in the database before the whitelist is configured are
+still applied, so it is suggested to only use the whitelist on instances that
+have not migrated the config to the database.
+
+Example:
+```elixir
+config :pleroma, :database_config_whitelist, [
+  {:pleroma, :instance},
+  {:pleroma, Pleroma.Web.Metadata},
+  {:auto_linker}
+]
+```
+
+### Multi-factor authentication -  :two_factor_authentication
+* `totp` - a list containing TOTP configuration
+  - `digits` - Determines the length of a one-time pass-code in characters. Defaults to 6 characters.
+  - `period` - a period for which the TOTP code will be valid in seconds. Defaults to 30 seconds.
+* `backup_codes` - a list containing backup codes configuration
+  - `number` - number of backup codes to generate.
+  - `length` - backup code length. Defaults to 16 characters.
 
 ## Restrict entities access for unauthenticated users
 
@@ -889,4 +949,9 @@ Restrict access for unauthenticated users to timelines (public and federate), us
   * `remote`
 * `activities` - statuses
   * `local`
-  * `remote`
+  * `remote`
+
+
+## Pleroma.Web.ApiSpec.CastAndValidate
+
+* `:strict` a boolean, enables strict input validation (useful in development, not recommended in production). Defaults to `false`.

docs/configuration/hardening.md

@@ -36,7 +36,7 @@ content-security-policy:
   default-src 'none';
   base-uri 'self';
   frame-ancestors 'none';
-  img-src 'self' data: https:;
+  img-src 'self' data: blob: https:;
   media-src 'self' https:;
   style-src 'self' 'unsafe-inline';
   font-src 'self';

docs/configuration/mrf.md

@@ -41,11 +41,15 @@ config :pleroma, :instance,
 
 Once `SimplePolicy` is enabled, you can configure various groups in the `:mrf_simple` config object. These groups are:
 
-* `media_removal`: Servers in this group will have media stripped from incoming messages.
-* `media_nsfw`: Servers in this group will have the #nsfw tag and sensitive setting injected into incoming messages which contain media.
 * `reject`: Servers in this group will have their messages rejected.
-* `federated_timeline_removal`: Servers in this group will have their messages unlisted from the public timelines by flipping the `to` and `cc` fields.
+* `accept`: If not empty, only messages from these instances will be accepted (whitelist federation).
+* `media_nsfw`: Servers in this group will have the #nsfw tag and sensitive setting injected into incoming messages which contain media.
+* `media_removal`: Servers in this group will have media stripped from incoming messages.
+* `avatar_removal`: Avatars from these servers will be stripped from incoming messages.
+* `banner_removal`: Banner images from these servers will be stripped from incoming messages.
 * `report_removal`: Servers in this group will have their reports (flags) rejected.
+* `federated_timeline_removal`: Servers in this group will have their messages unlisted from the public timelines by flipping the `to` and `cc` fields.
+* `reject_deletes`: Deletion requests will be rejected from these servers.
 
 Servers should be configured as lists.
 
@@ -113,7 +117,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.RewritePolicy do
 
   @impl true
   def describe do
-    {:ok, %{mrf_sample: %{content: "new message content"}}}`
+    {:ok, %{mrf_sample: %{content: "new message content"}}}
   end
 end
 ```

docs/dev.md

@@ -0,0 +1,23 @@
+This document contains notes and guidelines for Pleroma developers.
+
+# Authentication & Authorization
+
+## OAuth token-based authentication & authorization
+
+* Pleroma supports hierarchical OAuth scopes, just like Mastodon but with added granularity of admin scopes. For a reference, see [Mastodon OAuth scopes](https://docs.joinmastodon.org/api/oauth-scopes/).
+
+* It is important to either define OAuth scope restrictions or explicitly mark OAuth scope check as skipped, for every controller action. To define scopes, call `plug(Pleroma.Plugs.OAuthScopesPlug, %{scopes: [...]})`. To explicitly set OAuth scopes check skipped, call `plug(:skip_plug, Pleroma.Plugs.OAuthScopesPlug <when ...>)`.
+
+* In controllers, `use Pleroma.Web, :controller` will result in `action/2` (see `Pleroma.Web.controller/0` for definition) be called prior to actual controller action, and it'll perform security / privacy checks before passing control to actual controller action.
+
+  For routes with `:authenticated_api` pipeline, authentication & authorization are expected, thus `OAuthScopesPlug` will be run unless explicitly skipped (also `EnsureAuthenticatedPlug` will be executed immediately before action even if there was an early run to give an early error, since `OAuthScopesPlug` supports `:proceed_unauthenticated` option, and other plugs may support similar options as well).
+
+  For `:api` pipeline routes, it'll be verified whether `OAuthScopesPlug` was called or explicitly skipped, and if it was not then auth information will be dropped for request. Then `EnsurePublicOrAuthenticatedPlug` will be called to ensure that either the instance is not private or user is authenticated (unless explicitly skipped). Such automated checks help to prevent human errors and result in higher security / privacy for users.
+
+## [HTTP Basic Authentication](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Authorization)
+
+* With HTTP Basic Auth, OAuth scopes check is _not_ performed for any action (since password is provided during the auth, requester is able to obtain a token with full permissions anyways). `Pleroma.Plugs.AuthenticationPlug` and `Pleroma.Plugs.LegacyAuthenticationPlug` both call `Pleroma.Plugs.OAuthScopesPlug.skip_plug(conn)` when password is provided.
+
+## Auth-related configuration, OAuth consumer mode etc.
+
+See `Authentication` section of [`docs/configuration/cheatsheet.md`](docs/configuration/cheatsheet.md#authentication).

docs/installation/debian_based_en.md

@@ -7,13 +7,9 @@ This guide will assume you are on Debian Stretch. This guide should also work wi
 
 * `postgresql` (9.6+, Ubuntu 16.04 comes with 9.5, you can get a newer version from [here](https://www.postgresql.org/download/linux/ubuntu/))
 * `postgresql-contrib` (9.6+, same situtation as above)
-* `elixir` (1.5+, [install from here, Debian and Ubuntu ship older versions](https://elixir-lang.org/install.html#unix-and-unix-like) or use [asdf](https://github.com/asdf-vm/asdf) as the pleroma user)
+* `elixir` (1.8+, Follow the guide to install from the Erlang Solutions repo or use [asdf](https://github.com/asdf-vm/asdf) as the pleroma user)
 * `erlang-dev`
-* `erlang-tools`
+* `erlang-nox`
-* `erlang-parsetools`
-* `erlang-eldap`, if you want to enable ldap authenticator
-* `erlang-ssh`
-* `erlang-xmerl`
 * `git`
 * `build-essential`
 
@@ -50,7 +46,7 @@ sudo dpkg -i /tmp/erlang-solutions_1.0_all.deb
 
 ```shell
 sudo apt update
-sudo apt install elixir erlang-dev erlang-parsetools erlang-xmerl erlang-tools erlang-ssh
+sudo apt install elixir erlang-dev erlang-nox
 ```
 
 ### Install PleromaBE

docs/installation/debian_based_jp.md

@@ -10,21 +10,17 @@
 ### 必要なソフトウェア
 
 - PostgreSQL 9.6以上 (Ubuntu16.04では9.5しか提供されていないので，[](https://www.postgresql.org/download/linux/ubuntu/)こちらから新しいバージョンを入手してください)
-- postgresql-contrib 9.6以上 (同上)
+- `postgresql-contrib` 9.6以上 (同上)
-- Elixir 1.5 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください)
+- Elixir 1.8 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください)
-  - erlang-dev
+- `erlang-dev`
-- erlang-tools
+- `erlang-nox`
-- erlang-parsetools
+- `git`
-- erlang-eldap (LDAP認証を有効化するときのみ必要)
+- `build-essential`
-- erlang-ssh
-- erlang-xmerl
-- git
-- build-essential
 
 #### このガイドで利用している追加パッケージ
 
-- nginx (おすすめです。他のリバースプロキシを使う場合は、参考となる設定をこのリポジトリから探してください)
+- `nginx` (おすすめです。他のリバースプロキシを使う場合は、参考となる設定をこのリポジトリから探してください)
-- certbot (または何らかのLet's Encrypt向けACMEクライアント)
+- `certbot` (または何らかのLet's Encrypt向けACMEクライアント)
 
 ### システムを準備する
 
@@ -51,7 +47,7 @@ sudo dpkg -i /tmp/erlang-solutions_1.0_all.deb
 * ElixirとErlangをインストールします、
 ```
 sudo apt update
-sudo apt install elixir erlang-dev erlang-parsetools erlang-xmerl erlang-tools erlang-ssh
+sudo apt install elixir erlang-dev erlang-nox
 ```
 
 ### Pleroma BE (バックエンド) をインストールします

installation/init.d/pleroma

@@ -1,21 +1,45 @@
 #!/sbin/openrc-run
-
+supervisor=supervise-daemon
-# Requires OpenRC >= 0.35
-directory=/opt/pleroma
-
-command=/usr/bin/mix
-command_args="phx.server"
 command_user=pleroma:pleroma
 command_background=1
-
-export PORT=4000
-export MIX_ENV=prod
-
 # Ask process to terminate within 30 seconds, otherwise kill it
 retry="SIGTERM/30/SIGKILL/5"
-
 pidfile="/var/run/pleroma.pid"
+directory=/opt/pleroma
+healthcheck_delay=60
+healthcheck_timer=30
+
+: ${pleroma_port:-4000}
+
+# Needs OpenRC >= 0.42
+#respawn_max=0
+#respawn_delay=5
+
+# put pleroma_console=YES in /etc/conf.d/pleroma if you want to be able to
+# connect to pleroma via an elixir console
+if yesno "${pleroma_console}"; then
+	command=elixir
+	command_args="--name pleroma@127.0.0.1 --erl '-kernel inet_dist_listen_min 9001 inet_dist_listen_max 9001 inet_dist_use_interface {127,0,0,1}' -S mix phx.server"
+
+	start_post() {
+		einfo "You can get a console by using this command as pleroma's user:"
+		einfo "iex --name console@127.0.0.1 --remsh pleroma@127.0.0.1"
+	}
+else
+	command=/usr/bin/mix
+	command_args="phx.server"
+fi
+
+export MIX_ENV=prod
 
 depend() {
-    need nginx postgresql
+	need nginx postgresql
+}
+
+healthcheck() {
+	# put pleroma_health=YES in /etc/conf.d/pleroma if you want healthchecking
+	# and make sure you have curl installed
+	yesno "$pleroma_health" || return 0
+
+	curl -q "localhost:${pleroma_port}/api/pleroma/healthcheck"
 }

installation/pleroma-apache.conf

@@ -32,9 +32,8 @@ CustomLog ${APACHE_LOG_DIR}/access.log combined
 
 <VirtualHost *:443>
     SSLEngine on
-    SSLCertificateFile      /etc/letsencrypt/live/${servername}/cert.pem
+    SSLCertificateFile      /etc/letsencrypt/live/${servername}/fullchain.pem
     SSLCertificateKeyFile   /etc/letsencrypt/live/${servername}/privkey.pem
-    SSLCertificateChainFile /etc/letsencrypt/live/${servername}/fullchain.pem
 
     # Mozilla modern configuration, tweak to your needs
     SSLProtocol             all -SSLv3 -TLSv1 -TLSv1.1

lib/mix/tasks/pleroma/app.ex

@@ -0,0 +1,49 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.App do
+  @moduledoc File.read!("docs/administration/CLI_tasks/oauth_app.md")
+  use Mix.Task
+
+  import Mix.Pleroma
+
+  @shortdoc "Creates trusted OAuth App"
+
+  def run(["create" | options]) do
+    start_pleroma()
+
+    {opts, _} =
+      OptionParser.parse!(options,
+        strict: [name: :string, redirect_uri: :string, scopes: :string],
+        aliases: [n: :name, r: :redirect_uri, s: :scopes]
+      )
+
+    scopes =
+      if opts[:scopes] do
+        String.split(opts[:scopes], ",")
+      else
+        ["read", "write", "follow", "push"]
+      end
+
+    params = %{
+      client_name: opts[:name],
+      redirect_uris: opts[:redirect_uri],
+      trusted: true,
+      scopes: scopes
+    }
+
+    with {:ok, app} <- Pleroma.Web.OAuth.App.create(params) do
+      shell_info("#{app.client_name} successfully created:")
+      shell_info("App client_id: " <> app.client_id)
+      shell_info("App client_secret: " <> app.client_secret)
+    else
+      {:error, changeset} ->
+        shell_error("Creating failed:")
+
+        Enum.each(Pleroma.Web.OAuth.App.errors(changeset), fn {key, error} ->
+          shell_error("#{key}: #{error}")
+        end)
+    end
+  end
+end

lib/mix/tasks/pleroma/benchmark.ex

@@ -67,11 +67,51 @@ def run(["render_timeline", nickname | _] = args) do
           Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
             activities: activities,
             for: user,
-            as: :activity
+            as: :activity,
+            skip_relationships: true
           })
         end
       },
       inputs: inputs
     )
   end
+
+  def run(["adapters"]) do
+    start_pleroma()
+
+    :ok =
+      Pleroma.Gun.Conn.open(
+        "https://httpbin.org/stream-bytes/1500",
+        :gun_connections
+      )
+
+    Process.sleep(1_500)
+
+    Benchee.run(
+      %{
+        "Without conn and without pool" => fn ->
+          {:ok, %Tesla.Env{}} =
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
+              adapter: [pool: :no_pool, receive_conn: false]
+            )
+        end,
+        "Without conn and with pool" => fn ->
+          {:ok, %Tesla.Env{}} =
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
+              adapter: [receive_conn: false]
+            )
+        end,
+        "With reused conn and without pool" => fn ->
+          {:ok, %Tesla.Env{}} =
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
+              adapter: [pool: :no_pool]
+            )
+        end,
+        "With reused conn and with pool" => fn ->
+          {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500")
+        end
+      },
+      parallel: 10
+    )
+  end
 end

lib/mix/tasks/pleroma/digest.ex

@@ -1,5 +1,6 @@
 defmodule Mix.Tasks.Pleroma.Digest do
   use Mix.Task
+  import Mix.Pleroma
 
   @shortdoc "Manages digest emails"
   @moduledoc File.read!("docs/administration/CLI_tasks/digest.md")
@@ -22,12 +23,10 @@ def run(["test", nickname | opts]) do
     with %Swoosh.Email{} = email <- Pleroma.Emails.UserEmail.digest_email(patched_user) do
       {:ok, _} = Pleroma.Emails.Mailer.deliver(email)
 
-      Mix.shell().info("Digest email have been sent to #{nickname} (#{user.email})")
+      shell_info("Digest email have been sent to #{nickname} (#{user.email})")
     else
       _ ->
-        Mix.shell().info(
+        shell_info("Cound't find any mentions for #{nickname} since #{last_digest_emailed_at}")
-          "Cound't find any mentions for #{nickname} since #{last_digest_emailed_at}"
-        )
     end
   end
 end

lib/mix/tasks/pleroma/emoji.ex

@@ -4,18 +4,18 @@
 
 defmodule Mix.Tasks.Pleroma.Emoji do
   use Mix.Task
+  import Mix.Pleroma
 
   @shortdoc "Manages emoji packs"
   @moduledoc File.read!("docs/administration/CLI_tasks/emoji.md")
 
   def run(["ls-packs" | args]) do
-    Mix.Pleroma.start_pleroma()
+    start_pleroma()
-    Application.ensure_all_started(:hackney)
 
     {options, [], []} = parse_global_opts(args)
 
-    manifest =
+    url_or_path = options[:manifest] || default_manifest()
-      fetch_manifest(if options[:manifest], do: options[:manifest], else: default_manifest())
+    manifest = fetch_manifest(url_or_path)
 
     Enum.each(manifest, fn {name, info} ->
       to_print = [
@@ -36,14 +36,13 @@ def run(["ls-packs" | args]) do
   end
 
   def run(["get-packs" | args]) do
-    Mix.Pleroma.start_pleroma()
+    start_pleroma()
-    Application.ensure_all_started(:hackney)
 
     {options, pack_names, []} = parse_global_opts(args)
 
-    manifest_url = if options[:manifest], do: options[:manifest], else: default_manifest()
+    url_or_path = options[:manifest] || default_manifest()
 
-    manifest = fetch_manifest(manifest_url)
+    manifest = fetch_manifest(url_or_path)
 
     for pack_name <- pack_names do
       if Map.has_key?(manifest, pack_name) do
@@ -76,7 +75,10 @@ def run(["get-packs" | args]) do
         end
 
         # The url specified in files should be in the same directory
-        files_url = Path.join(Path.dirname(manifest_url), pack["files"])
+        files_url =
+          url_or_path
+          |> Path.dirname()
+          |> Path.join(pack["files"])
 
         IO.puts(
           IO.ANSI.format([
@@ -134,38 +136,51 @@ def run(["get-packs" | args]) do
     end
   end
 
-  def run(["gen-pack", src]) do
+  def run(["gen-pack" | args]) do
-    Application.ensure_all_started(:hackney)
+    start_pleroma()
 
-    proposed_name = Path.basename(src) |> Path.rootname()
+    {opts, [src], []} =
-    name = String.trim(IO.gets("Pack name [#{proposed_name}]: "))
+      OptionParser.parse(
-    # If there's no name, use the default one
+        args,
-    name = if String.length(name) > 0, do: name, else: proposed_name
+        strict: [
-
+          name: :string,
-    license = String.trim(IO.gets("License: "))
+          license: :string,
-    homepage = String.trim(IO.gets("Homepage: "))
+          homepage: :string,
-    description = String.trim(IO.gets("Description: "))
+          description: :string,
-
+          files: :string,
-    proposed_files_name = "#{name}.json"
+          extensions: :string
-    files_name = String.trim(IO.gets("Save file list to [#{proposed_files_name}]: "))
+        ]
-    files_name = if String.length(files_name) > 0, do: files_name, else: proposed_files_name
-
-    default_exts = [".png", ".gif"]
-    default_exts_str = Enum.join(default_exts, " ")
-
-    exts =
-      String.trim(
-        IO.gets("Emoji file extensions (separated with spaces) [#{default_exts_str}]: ")
       )
 
+    proposed_name = Path.basename(src) |> Path.rootname()
+    name = get_option(opts, :name, "Pack name:", proposed_name)
+    license = get_option(opts, :license, "License:")
+    homepage = get_option(opts, :homepage, "Homepage:")
+    description = get_option(opts, :description, "Description:")
+
+    proposed_files_name = "#{name}_files.json"
+    files_name = get_option(opts, :files, "Save file list to:", proposed_files_name)
+
+    default_exts = [".png", ".gif"]
+
+    custom_exts =
+      get_option(
+        opts,
+        :extensions,
+        "Emoji file extensions (separated with spaces):",
+        Enum.join(default_exts, " ")
+      )
+      |> String.split(" ", trim: true)
+
     exts =
-      if String.length(exts) > 0 do
+      if MapSet.equal?(MapSet.new(default_exts), MapSet.new(custom_exts)) do
-        String.split(exts, " ")
-        |> Enum.filter(fn e -> e |> String.trim() |> String.length() > 0 end)
-      else
         default_exts
+      else
+        custom_exts
       end
 
+    IO.puts("Using #{Enum.join(exts, " ")} extensions")
+
     IO.puts("Downloading the pack and generating SHA256")
 
     binary_archive = Tesla.get!(client(), src).body
@@ -195,14 +210,16 @@ def run(["gen-pack", src]) do
     IO.puts("""
 
     #{files_name} has been created and contains the list of all found emojis in the pack.
-    Please review the files in the remove those not needed.
+    Please review the files in the pack and remove those not needed.
     """)
 
-    if File.exists?("index.json") do
+    pack_file = "#{name}.json"
-      existing_data = File.read!("index.json") |> Jason.decode!()
+
+    if File.exists?(pack_file) do
+      existing_data = File.read!(pack_file) |> Jason.decode!()
 
       File.write!(
-        "index.json",
+        pack_file,
         Jason.encode!(
           Map.merge(
             existing_data,
@@ -212,11 +229,11 @@ def run(["gen-pack", src]) do
         )
       )
 
-      IO.puts("index.json file has been update with the #{name} pack")
+      IO.puts("#{pack_file} has been updated with the #{name} pack")
     else
-      File.write!("index.json", Jason.encode!(pack_json, pretty: true))
+      File.write!(pack_file, Jason.encode!(pack_json, pretty: true))
 
-      IO.puts("index.json has been created with the #{name} pack")
+      IO.puts("#{pack_file} has been created with the #{name} pack")
     end
   end
 

lib/mix/tasks/pleroma/user.ex

@@ -8,6 +8,8 @@ defmodule Mix.Tasks.Pleroma.User do
   alias Ecto.Changeset
   alias Pleroma.User
   alias Pleroma.UserInviteToken
+  alias Pleroma.Web.ActivityPub.Builder
+  alias Pleroma.Web.ActivityPub.Pipeline
 
   @shortdoc "Manages Pleroma users"
   @moduledoc File.read!("docs/administration/CLI_tasks/user.md")
@@ -96,8 +98,9 @@ def run(["new", nickname, email | rest]) do
   def run(["rm", nickname]) do
     start_pleroma()
 
-    with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
+    with %User{local: true} = user <- User.get_cached_by_nickname(nickname),
-      User.perform(:delete, user)
+         {:ok, delete_data, _} <- Builder.delete(user, user.ap_id),
+         {:ok, _delete, _} <- Pipeline.common_pipeline(delete_data, local: true) do
       shell_info("User #{nickname} deleted.")
     else
       _ -> shell_error("No local user #{nickname}")

lib/pleroma/activity.ex

@@ -27,17 +27,13 @@ defmodule Pleroma.Activity do
   # https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19
   @mastodon_notification_types %{
     "Create" => "mention",
-    "Follow" => "follow",
+    "Follow" => ["follow", "follow_request"],
     "Announce" => "reblog",
     "Like" => "favourite",
     "Move" => "move",
     "EmojiReact" => "pleroma:emoji_reaction"
   }
 
-  @mastodon_to_ap_notification_types for {k, v} <- @mastodon_notification_types,
-                                         into: %{},
-                                         do: {v, k}
-
   schema "activities" do
     field(:data, :map)
     field(:local, :boolean, default: true)
@@ -95,6 +91,17 @@ def with_preloaded_object(query, join_type \\ :inner) do
     |> preload([activity, object: object], object: object)
   end
 
+  # Note: applies to fake activities (ActivityPub.Utils.get_notified_from_object/1 etc.)
+  def user_actor(%Activity{actor: nil}), do: nil
+
+  def user_actor(%Activity{} = activity) do
+    with %User{} <- activity.user_actor do
+      activity.user_actor
+    else
+      _ -> User.get_cached_by_ap_id(activity.actor)
+    end
+  end
+
   def with_joined_user_actor(query, join_type \\ :inner) do
     join(query, join_type, [activity], u in User,
       on: u.ap_id == activity.actor,
@@ -280,15 +287,43 @@ defp purge_web_resp_cache(%Activity{} = activity) do
 
   defp purge_web_resp_cache(nil), do: nil
 
-  for {ap_type, type} <- @mastodon_notification_types do
+  def follow_accepted?(
+        %Activity{data: %{"type" => "Follow", "object" => followed_ap_id}} = activity
+      ) do
+    with %User{} = follower <- Activity.user_actor(activity),
+         %User{} = followed <- User.get_cached_by_ap_id(followed_ap_id) do
+      Pleroma.FollowingRelationship.following?(follower, followed)
+    else
+      _ -> false
+    end
+  end
+
+  def follow_accepted?(_), do: false
+
+  @spec mastodon_notification_type(Activity.t()) :: String.t() | nil
+
+  for {ap_type, type} <- @mastodon_notification_types, not is_list(type) do
     def mastodon_notification_type(%Activity{data: %{"type" => unquote(ap_type)}}),
       do: unquote(type)
   end
 
+  def mastodon_notification_type(%Activity{data: %{"type" => "Follow"}} = activity) do
+    if follow_accepted?(activity) do
+      "follow"
+    else
+      "follow_request"
+    end
+  end
+
   def mastodon_notification_type(%Activity{}), do: nil
 
+  @spec from_mastodon_notification_type(String.t()) :: String.t() | nil
+  @doc "Converts Mastodon notification type to AR activity type"
   def from_mastodon_notification_type(type) do
-    Map.get(@mastodon_to_ap_notification_types, type)
+    with {k, _v} <-
+           Enum.find(@mastodon_notification_types, fn {_k, v} -> type in List.wrap(v) end) do
+      k
+    end
   end
 
   def all_by_actor_and_id(actor, status_ids \\ [])

lib/pleroma/activity/queries.ex

@@ -35,6 +35,13 @@ def by_author(query \\ Activity, %User{ap_id: ap_id}) do
     from(a in query, where: a.actor == ^ap_id)
   end
 
+  def find_by_object_ap_id(activities, object_ap_id) do
+    Enum.find(
+      activities,
+      &(object_ap_id in [is_map(&1.data["object"]) && &1.data["object"]["id"], &1.data["object"]])
+    )
+  end
+
   @spec by_object_id(query, String.t() | [String.t()]) :: query
   def by_object_id(query \\ Activity, object_id)
 

lib/pleroma/application.ex

@@ -3,8 +3,12 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Application do
-  import Cachex.Spec
   use Application
+
+  import Cachex.Spec
+
+  alias Pleroma.Config
+
   require Logger
 
   @name Mix.Project.config()[:name]
@@ -18,9 +22,9 @@ def named_version, do: @name <> " " <> @version
   def repository, do: @repository
 
   def user_agent do
-    case Pleroma.Config.get([:http, :user_agent], :default) do
+    case Config.get([:http, :user_agent], :default) do
       :default ->
-        info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>"
+        info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>"
         named_version() <> "; " <> info
 
       custom ->
@@ -33,27 +37,50 @@ def user_agent do
   def start(_type, _args) do
     Pleroma.Config.Holder.save_default()
     Pleroma.HTML.compile_scrubbers()
-    Pleroma.Config.DeprecationWarnings.warn()
+    Config.DeprecationWarnings.warn()
     Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
     Pleroma.Repo.check_migrations_applied!()
     setup_instrumenters()
     load_custom_modules()
 
+    adapter = Application.get_env(:tesla, :adapter)
+
+    if adapter == Tesla.Adapter.Gun do
+      if version = Pleroma.OTPVersion.version() do
+        [major, minor] =
+          version
+          |> String.split(".")
+          |> Enum.map(&String.to_integer/1)
+          |> Enum.take(2)
+
+        if (major == 22 and minor < 2) or major < 22 do
+          raise "
+            !!!OTP VERSION WARNING!!!
+            You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. Please update your Erlang/OTP to at least 22.2.
+            "
+        end
+      else
+        raise "
+          !!!OTP VERSION WARNING!!!
+          To support correct handling of unordered certificates chains - OTP version must be > 22.2.
+          "
+      end
+    end
+
     # Define workers and child supervisors to be supervised
     children =
       [
         Pleroma.Repo,
-        Pleroma.Config.TransferTask,
+        Config.TransferTask,
         Pleroma.Emoji,
-        Pleroma.Captcha,
         Pleroma.Plugs.RateLimiter.Supervisor
       ] ++
         cachex_children() ++
-        hackney_pool_children() ++
+        http_children(adapter, @env) ++
         [
           Pleroma.Stats,
           Pleroma.JobQueueMonitor,
-          {Oban, Pleroma.Config.get(Oban)}
+          {Oban, Config.get(Oban)}
         ] ++
         task_children(@env) ++
         streamer_child(@env) ++
@@ -70,7 +97,7 @@ def start(_type, _args) do
   end
 
   def load_custom_modules do
-    dir = Pleroma.Config.get([:modules, :runtime_dir])
+    dir = Config.get([:modules, :runtime_dir])
 
     if dir && File.exists?(dir) do
       dir
@@ -111,20 +138,6 @@ defp setup_instrumenters do
     Pleroma.Web.Endpoint.Instrumenter.setup()
   end
 
-  def enabled_hackney_pools do
-    [:media] ++
-      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
-        [:federation]
-      else
-        []
-      end ++
-      if Pleroma.Config.get([Pleroma.Upload, :proxy_remote]) do
-        [:upload]
-      else
-        []
-      end
-  end
-
   defp cachex_children do
     [
       build_cachex("used_captcha", ttl_interval: seconds_valid_interval()),
@@ -146,7 +159,7 @@ defp idempotency_expiration,
     do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))
 
   defp seconds_valid_interval,
-    do: :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid]))
+    do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid]))
 
   defp build_cachex(type, opts),
     do: %{
@@ -155,12 +168,19 @@ defp build_cachex(type, opts),
       type: :worker
     }
 
-  defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled])
+  defp chat_enabled?, do: Config.get([:chat, :enabled])
 
-  defp streamer_child(:test), do: []
+  defp streamer_child(env) when env in [:test, :benchmark], do: []
 
   defp streamer_child(_) do
-    [Pleroma.Web.Streamer.supervisor()]
+    [
+      {Registry,
+       [
+         name: Pleroma.Web.Streamer.registry(),
+         keys: :duplicate,
+         partitions: System.schedulers_online()
+       ]}
+    ]
   end
 
   defp chat_child(_env, true) do
@@ -169,13 +189,6 @@ defp chat_child(_env, true) do
 
   defp chat_child(_, _), do: []
 
-  defp hackney_pool_children do
-    for pool <- enabled_hackney_pools() do
-      options = Pleroma.Config.get([:hackney_pools, pool])
-      :hackney_pool.child_spec(pool, options)
-    end
-  end
-
   defp task_children(:test) do
     [
       %{
@@ -200,4 +213,31 @@ defp task_children(_) do
       }
     ]
   end
+
+  # start hackney and gun pools in tests
+  defp http_children(_, :test) do
+    hackney_options = Config.get([:hackney_pools, :federation])
+    hackney_pool = :hackney_pool.child_spec(:federation, hackney_options)
+    [hackney_pool, Pleroma.Pool.Supervisor]
+  end
+
+  defp http_children(Tesla.Adapter.Hackney, _) do
+    pools = [:federation, :media]
+
+    pools =
+      if Config.get([Pleroma.Upload, :proxy_remote]) do
+        [:upload | pools]
+      else
+        pools
+      end
+
+    for pool <- pools do
+      options = Config.get([:hackney_pools, pool])
+      :hackney_pool.child_spec(pool, options)
+    end
+  end
+
+  defp http_children(Tesla.Adapter.Gun, _), do: [Pleroma.Pool.Supervisor]
+
+  defp http_children(_, _), do: []
 end

lib/pleroma/bbs/authenticator.ex

@@ -4,7 +4,7 @@
 
 defmodule Pleroma.BBS.Authenticator do
   use Sshd.PasswordAuthenticator
-  alias Comeonin.Pbkdf2
+  alias Pleroma.Plugs.AuthenticationPlug
   alias Pleroma.User
 
   def authenticate(username, password) do
@@ -12,7 +12,7 @@ def authenticate(username, password) do
     password = to_string(password)
 
     with %User{} = user <- User.get_by_nickname(username) do
-      Pbkdf2.checkpw(password, user.password_hash)
+      AuthenticationPlug.checkpw(password, user.password_hash)
     else
       _e -> false
     end

lib/pleroma/bbs/handler.ex

@@ -66,7 +66,7 @@ def handle_command(%{user: user} = state, "r " <> text) do
 
     with %Activity{} <- Activity.get_by_id(activity_id),
          {:ok, _activity} <-
-           CommonAPI.post(user, %{"status" => rest, "in_reply_to_status_id" => activity_id}) do
+           CommonAPI.post(user, %{status: rest, in_reply_to_status_id: activity_id}) do
       IO.puts("Replied!")
     else
       _e -> IO.puts("Could not reply...")
@@ -78,7 +78,7 @@ def handle_command(%{user: user} = state, "r " <> text) do
   def handle_command(%{user: user} = state, "p " <> text) do
     text = String.trim(text)
 
-    with {:ok, _activity} <- CommonAPI.post(user, %{"status" => text}) do
+    with {:ok, _activity} <- CommonAPI.post(user, %{status: text}) do
       IO.puts("Posted!")
     else
       _e -> IO.puts("Could not post...")

lib/pleroma/captcha/captcha.ex

@@ -3,53 +3,22 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Captcha do
-  import Pleroma.Web.Gettext
-
   alias Calendar.DateTime
   alias Plug.Crypto.KeyGenerator
   alias Plug.Crypto.MessageEncryptor
 
-  use GenServer
-
-  @doc false
-  def start_link(_) do
-    GenServer.start_link(__MODULE__, [], name: __MODULE__)
-  end
-
-  @doc false
-  def init(_) do
-    {:ok, nil}
-  end
-
   @doc """
   Ask the configured captcha service for a new captcha
   """
   def new do
-    GenServer.call(__MODULE__, :new)
+    if not enabled?() do
-  end
+      %{type: :none}
-
-  @doc """
-  Ask the configured captcha service to validate the captcha
-  """
-  def validate(token, captcha, answer_data) do
-    GenServer.call(__MODULE__, {:validate, token, captcha, answer_data})
-  end
-
-  @doc false
-  def handle_call(:new, _from, state) do
-    enabled = Pleroma.Config.get([__MODULE__, :enabled])
-
-    if !enabled do
-      {:reply, %{type: :none}, state}
     else
       new_captcha = method().new()
 
-      secret_key_base = Pleroma.Config.get!([Pleroma.Web.Endpoint, :secret_key_base])
-
       # This make salt a little different for two keys
-      token = new_captcha[:token]
+      {secret, sign_secret} = secret_pair(new_captcha[:token])
-      secret = KeyGenerator.generate(secret_key_base, token <> "_encrypt")
+
-      sign_secret = KeyGenerator.generate(secret_key_base, token <> "_sign")
       # Basically copy what Phoenix.Token does here, add the time to
       # the actual data and make it a binary to then encrypt it
       encrypted_captcha_answer =
@@ -60,55 +29,73 @@ def handle_call(:new, _from, state) do
         |> :erlang.term_to_binary()
         |> MessageEncryptor.encrypt(secret, sign_secret)
 
-      {
+      # Replace the answer with the encrypted answer
-        :reply,
+      %{new_captcha | answer_data: encrypted_captcha_answer}
-        # Replace the answer with the encrypted answer
-        %{new_captcha | answer_data: encrypted_captcha_answer},
-        state
-      }
     end
   end
 
-  @doc false
+  @doc """
-  def handle_call({:validate, token, captcha, answer_data}, _from, state) do
+  Ask the configured captcha service to validate the captcha
+  """
+  def validate(token, captcha, answer_data) do
+    with {:ok, %{at: at, answer_data: answer_md5}} <- validate_answer_data(token, answer_data),
+         :ok <- validate_expiration(at),
+         :ok <- validate_usage(token),
+         :ok <- method().validate(token, captcha, answer_md5),
+         {:ok, _} <- mark_captcha_as_used(token) do
+      :ok
+    end
+  end
+
+  def enabled?, do: Pleroma.Config.get([__MODULE__, :enabled], false)
+
+  defp seconds_valid, do: Pleroma.Config.get!([__MODULE__, :seconds_valid])
+
+  defp secret_pair(token) do
     secret_key_base = Pleroma.Config.get!([Pleroma.Web.Endpoint, :secret_key_base])
     secret = KeyGenerator.generate(secret_key_base, token <> "_encrypt")
     sign_secret = KeyGenerator.generate(secret_key_base, token <> "_sign")
 
+    {secret, sign_secret}
+  end
+
+  defp validate_answer_data(token, answer_data) do
+    {secret, sign_secret} = secret_pair(token)
+
+    with false <- is_nil(answer_data),
+         {:ok, data} <- MessageEncryptor.decrypt(answer_data, secret, sign_secret),
+         %{at: at, answer_data: answer_md5} <- :erlang.binary_to_term(data) do
+      {:ok, %{at: at, answer_data: answer_md5}}
+    else
+      _ -> {:error, :invalid_answer_data}
+    end
+  end
+
+  defp validate_expiration(created_at) do
     # If the time found is less than (current_time-seconds_valid) then the time has already passed
     # Later we check that the time found is more than the presumed invalidatation time, that means
     # that the data is still valid and the captcha can be checked
-    seconds_valid = Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid])
-    valid_if_after = DateTime.subtract!(DateTime.now_utc(), seconds_valid)
 
-    result =
+    valid_if_after = DateTime.subtract!(DateTime.now_utc(), seconds_valid())
-      with false <- is_nil(answer_data),
-           {:ok, data} <- MessageEncryptor.decrypt(answer_data, secret, sign_secret),
-           %{at: at, answer_data: answer_md5} <- :erlang.binary_to_term(data) do
-        try do
-          if DateTime.before?(at, valid_if_after),
-            do: throw({:error, dgettext("errors", "CAPTCHA expired")})
 
-          if not is_nil(Cachex.get!(:used_captcha_cache, token)),
+    if DateTime.before?(created_at, valid_if_after) do
-            do: throw({:error, dgettext("errors", "CAPTCHA already used")})
+      {:error, :expired}
+    else
+      :ok
+    end
+  end
 
-          res = method().validate(token, captcha, answer_md5)
+  defp validate_usage(token) do
-          # Throw if an error occurs
+    if is_nil(Cachex.get!(:used_captcha_cache, token)) do
-          if res != :ok, do: throw(res)
+      :ok
+    else
+      {:error, :already_used}
+    end
+  end
 
-          # Mark this captcha as used
+  defp mark_captcha_as_used(token) do
-          {:ok, _} =
+    ttl = seconds_valid() |> :timer.seconds()
-            Cachex.put(:used_captcha_cache, token, true, ttl: :timer.seconds(seconds_valid))
+    Cachex.put(:used_captcha_cache, token, true, ttl: ttl)
-
-          :ok
-        catch
-          :throw, e -> e
-        end
-      else
-        _ -> {:error, dgettext("errors", "Invalid answer data")}
-      end
-
-    {:reply, result, state}
   end
 
   defp method, do: Pleroma.Config.get!([__MODULE__, :method])

lib/pleroma/captcha/kocaptcha.ex

@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Captcha.Kocaptcha do
-  import Pleroma.Web.Gettext
   alias Pleroma.Captcha.Service
   @behaviour Service
 
@@ -13,7 +12,7 @@ def new do
 
     case Tesla.get(endpoint <> "/new") do
       {:error, _} ->
-        %{error: dgettext("errors", "Kocaptcha service unavailable")}
+        %{error: :kocaptcha_service_unavailable}
 
       {:ok, res} ->
         json_resp = Jason.decode!(res.body)
@@ -33,6 +32,6 @@ def validate(_token, captcha, answer_data) do
     if not is_nil(captcha) and
          :crypto.hash(:md5, captcha) |> Base.encode16() == String.upcase(answer_data),
        do: :ok,
-       else: {:error, dgettext("errors", "Invalid CAPTCHA")}
+       else: {:error, :invalid}
   end
 end

lib/pleroma/captcha/native.ex

@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Captcha.Native do
-  import Pleroma.Web.Gettext
   alias Pleroma.Captcha.Service
   @behaviour Service
 
@@ -11,7 +10,7 @@ defmodule Pleroma.Captcha.Native do
   def new do
     case Captcha.get() do
       :error ->
-        %{error: dgettext("errors", "Captcha error")}
+        %{error: :captcha_error}
 
       {:ok, answer_data, img_binary} ->
         %{
@@ -25,7 +24,7 @@ def new do
 
   @impl Service
   def validate(_token, captcha, captcha) when not is_nil(captcha), do: :ok
-  def validate(_token, _captcha, _answer), do: {:error, dgettext("errors", "Invalid CAPTCHA")}
+  def validate(_token, _captcha, _answer), do: {:error, :invalid}
 
   defp token do
     10

lib/pleroma/config/loader.ex

@@ -47,7 +47,7 @@ defp filter(configs) do
   @spec filter_group(atom(), keyword()) :: keyword()
   def filter_group(group, configs) do
     Enum.reject(configs[group], fn {key, _v} ->
-      key in @reject_keys or (group == :phoenix and key == :serve_endpoints)
+      key in @reject_keys or (group == :phoenix and key == :serve_endpoints) or group == :postgrex
     end)
   end
 end

lib/pleroma/config/transfer_task.ex

@@ -5,6 +5,7 @@
 defmodule Pleroma.Config.TransferTask do
   use Task
 
+  alias Pleroma.Config
   alias Pleroma.ConfigDB
   alias Pleroma.Repo
 
@@ -18,7 +19,9 @@ defmodule Pleroma.Config.TransferTask do
     {:pleroma, Oban},
     {:pleroma, :rate_limit},
     {:pleroma, :markup},
-    {:plerome, :streamer}
+    {:pleroma, :streamer},
+    {:pleroma, :pools},
+    {:pleroma, :connections_pool}
   ]
 
   @reboot_time_subkeys [
@@ -32,45 +35,44 @@ defmodule Pleroma.Config.TransferTask do
     {:pleroma, :gopher, [:enabled]}
   ]
 
-  @reject [nil, :prometheus]
-
   def start_link(_) do
     load_and_update_env()
-    if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo)
+    if Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo)
     :ignore
   end
 
-  @spec load_and_update_env([ConfigDB.t()]) :: :ok | false
+  @spec load_and_update_env([ConfigDB.t()], boolean()) :: :ok
-  def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do
+  def load_and_update_env(deleted_settings \\ [], restart_pleroma? \\ true) do
-    with {:configurable, true} <-
+    with {_, true} <- {:configurable, Config.get(:configurable_from_database)} do
-           {:configurable, Pleroma.Config.get(:configurable_from_database)},
-         true <- Ecto.Adapters.SQL.table_exists?(Repo, "config"),
-         started_applications <- Application.started_applications() do
       # We need to restart applications for loaded settings take effect
 
-      in_db = Repo.all(ConfigDB)
+      {logger, other} =
+        (Repo.all(ConfigDB) ++ deleted_settings)
+        |> Enum.map(&transform_and_merge/1)
+        |> Enum.split_with(fn {group, _, _, _} -> group in [:logger, :quack] end)
 
-      with_deleted = in_db ++ deleted
+      logger
+      |> Enum.sort()
+      |> Enum.each(&configure/1)
 
-      reject_for_restart = if restart_pleroma?, do: @reject, else: [:pleroma | @reject]
+      started_applications = Application.started_applications()
 
-      applications =
+      # TODO: some problem with prometheus after restart!
-        with_deleted
+      reject = [nil, :prometheus, :postgrex]
-        |> Enum.map(&merge_and_update(&1))
-        |> Enum.uniq()
-        # TODO: some problem with prometheus after restart!
-        |> Enum.reject(&(&1 in reject_for_restart))
 
-      # to be ensured that pleroma will be restarted last
+      reject =
-      applications =
+        if restart_pleroma? do
-        if :pleroma in applications do
+          reject
-          List.delete(applications, :pleroma) ++ [:pleroma]
         else
-          Restarter.Pleroma.rebooted()
+          [:pleroma | reject]
-          applications
         end
 
-      Enum.each(applications, &restart(started_applications, &1, Pleroma.Config.get(:env)))
+      other
+      |> Enum.map(&update/1)
+      |> Enum.uniq()
+      |> Enum.reject(&(&1 in reject))
+      |> maybe_set_pleroma_last()
+      |> Enum.each(&restart(started_applications, &1, Config.get(:env)))
 
       :ok
     else
@@ -78,51 +80,83 @@ def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do
     end
   end
 
-  defp merge_and_update(setting) do
+  defp maybe_set_pleroma_last(apps) do
-    try do
+    # to be ensured that pleroma will be restarted last
-      key = ConfigDB.from_string(setting.key)
+    if :pleroma in apps do
-      group = ConfigDB.from_string(setting.group)
+      apps
+      |> List.delete(:pleroma)
+      |> List.insert_at(-1, :pleroma)
+    else
+      Restarter.Pleroma.rebooted()
+      apps
+    end
+  end
 
-      default = Pleroma.Config.Holder.default_config(group, key)
+  defp transform_and_merge(%{group: group, key: key, value: value} = setting) do
-      value = ConfigDB.from_binary(setting.value)
+    group = ConfigDB.from_string(group)
+    key = ConfigDB.from_string(key)
+    value = ConfigDB.from_binary(value)
 
-      merged_value =
+    default = Config.Holder.default_config(group, key)
-        if Ecto.get_meta(setting, :state) == :deleted do
-          default
-        else
-          if can_be_merged?(default, value) do
-            ConfigDB.merge_group(group, key, default, value)
-          else
-            value
-          end
-        end
 
-      :ok = update_env(group, key, merged_value)
+    merged =
-
+      cond do
-      if group != :logger do
+        Ecto.get_meta(setting, :state) == :deleted -> default
-        if group != :pleroma or pleroma_need_restart?(group, key, value) do
+        can_be_merged?(default, value) -> ConfigDB.merge_group(group, key, default, value)
-          group
+        true -> value
-        end
-      else
-        # change logger configuration in runtime, without restart
-        if Keyword.keyword?(merged_value) and
-             key not in [:compile_time_application, :backends, :compile_time_purge_matching] do
-          Logger.configure_backend(key, merged_value)
-        else
-          Logger.configure([{key, merged_value}])
-        end
-
-        nil
       end
+
+    {group, key, value, merged}
+  end
+
+  # change logger configuration in runtime, without restart
+  defp configure({:quack, key, _, merged}) do
+    Logger.configure_backend(Quack.Logger, [{key, merged}])
+    :ok = update_env(:quack, key, merged)
+  end
+
+  defp configure({_, :backends, _, merged}) do
+    # removing current backends
+    Enum.each(Application.get_env(:logger, :backends), &Logger.remove_backend/1)
+
+    Enum.each(merged, &Logger.add_backend/1)
+
+    :ok = update_env(:logger, :backends, merged)
+  end
+
+  defp configure({_, key, _, merged}) when key in [:console, :ex_syslogger] do
+    merged =
+      if key == :console do
+        put_in(merged[:format], merged[:format] <> "\n")
+      else
+        merged
+      end
+
+    backend =
+      if key == :ex_syslogger,
+        do: {ExSyslogger, :ex_syslogger},
+        else: key
+
+    Logger.configure_backend(backend, merged)
+    :ok = update_env(:logger, key, merged)
+  end
+
+  defp configure({_, key, _, merged}) do
+    Logger.configure([{key, merged}])
+    :ok = update_env(:logger, key, merged)
+  end
+
+  defp update({group, key, value, merged}) do
+    try do
+      :ok = update_env(group, key, merged)
+
+      if group != :pleroma or pleroma_need_restart?(group, key, value), do: group
     rescue
       error ->
         error_msg =
-          "updating env causes error, group: " <>
+          "updating env causes error, group: #{inspect(group)}, key: #{inspect(key)}, value: #{
-            inspect(setting.group) <>
+            inspect(value)
-            " key: " <>
+          } error: #{inspect(error)}"
-            inspect(setting.key) <>
-            " value: " <>
-            inspect(ConfigDB.from_binary(setting.value)) <> " error: " <> inspect(error)
 
         Logger.warn(error_msg)
 
@@ -130,6 +164,9 @@ defp merge_and_update(setting) do
     end
   end
 
+  defp update_env(group, key, nil), do: Application.delete_env(group, key)
+  defp update_env(group, key, value), do: Application.put_env(group, key, value)
+
   @spec pleroma_need_restart?(atom(), atom(), any()) :: boolean()
   def pleroma_need_restart?(group, key, value) do
     group_and_key_need_reboot?(group, key) or group_and_subkey_need_reboot?(group, key, value)
@@ -147,9 +184,6 @@ defp group_and_subkey_need_reboot?(group, key, value) do
       end)
   end
 
-  defp update_env(group, key, nil), do: Application.delete_env(group, key)
-  defp update_env(group, key, value), do: Application.put_env(group, key, value)
-
   defp restart(_, :pleroma, env), do: Restarter.Pleroma.restart_after_boot(env)
 
   defp restart(started_applications, app, _) do

lib/pleroma/constants.ex

@@ -17,7 +17,13 @@ defmodule Pleroma.Constants do
       "announcement_count",
       "emoji",
       "context_id",
-      "deleted_activity_id"
+      "deleted_activity_id",
+      "pleroma_internal"
     ]
   )
+
+  const(static_only_files,
+    do:
+      ~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc)
+  )
 end

lib/pleroma/conversation/participation.ex

@@ -128,22 +128,19 @@ def for_user(user, params \\ %{}) do
     |> Pleroma.Pagination.fetch_paginated(params)
   end
 
-  def restrict_recipients(query, user, %{"recipients" => user_ids}) do
+  def restrict_recipients(query, user, %{recipients: user_ids}) do
-    user_ids =
+    user_binary_ids =
       [user.id | user_ids]
       |> Enum.uniq()
-      |> Enum.reduce([], fn user_id, acc ->
+      |> User.binary_id()
-        {:ok, user_id} = FlakeId.Ecto.CompatType.dump(user_id)
-        [user_id | acc]
-      end)
 
     conversation_subquery =
       __MODULE__
       |> group_by([p], p.conversation_id)
       |> having(
         [p],
-        count(p.user_id) == ^length(user_ids) and
+        count(p.user_id) == ^length(user_binary_ids) and
-          fragment("array_agg(?) @> ?", p.user_id, ^user_ids)
+          fragment("array_agg(?) @> ?", p.user_id, ^user_binary_ids)
       )
       |> select([p], %{id: p.conversation_id})
 
@@ -175,7 +172,7 @@ def for_user_with_last_activity_id(user, params \\ %{}) do
         | last_activity_id: activity_id
       }
     end)
-    |> Enum.filter(& &1.last_activity_id)
+    |> Enum.reject(&is_nil(&1.last_activity_id))
   end
 
   def get(_, _ \\ [])

lib/pleroma/docs/json.ex

@@ -18,7 +18,6 @@ def compile do
     with config <- Pleroma.Config.Loader.read("config/description.exs") do
       config[:pleroma][:config_description]
       |> Pleroma.Docs.Generator.convert_to_strings()
-      |> Jason.encode!()
     end
   end
 end

lib/pleroma/ecto_enums.ex

@@ -4,10 +4,16 @@
 
 import EctoEnum
 
-defenum(UserRelationshipTypeEnum,
+defenum(Pleroma.UserRelationship.Type,
   block: 1,
   mute: 2,
   reblog_mute: 3,
   notification_mute: 4,
   inverse_subscription: 5
 )
+
+defenum(Pleroma.FollowingRelationship.State,
+  follow_pending: 1,
+  follow_accept: 2,
+  follow_reject: 3
+)

lib/pleroma/emoji/formatter.ex

@@ -38,22 +38,14 @@ def demojify(text) do
 
   def demojify(text, nil), do: text
 
-  @doc "Outputs a list of the emoji-shortcodes in a text"
-  def get_emoji(text) when is_binary(text) do
-    Enum.filter(Emoji.get_all(), fn {emoji, %Emoji{}} ->
-      String.contains?(text, ":#{emoji}:")
-    end)
-  end
-
-  def get_emoji(_), do: []
-
   @doc "Outputs a list of the emoji-Maps in a text"
   def get_emoji_map(text) when is_binary(text) do
-    get_emoji(text)
+    Emoji.get_all()
+    |> Enum.filter(fn {emoji, %Emoji{}} -> String.contains?(text, ":#{emoji}:") end)
     |> Enum.reduce(%{}, fn {name, %Emoji{file: file}}, acc ->
       Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
     end)
   end
 
-  def get_emoji_map(_), do: []
+  def get_emoji_map(_), do: %{}
 end

lib/pleroma/emoji/pack.ex

@@ -0,0 +1,507 @@
+defmodule Pleroma.Emoji.Pack do
+  @derive {Jason.Encoder, only: [:files, :pack]}
+  defstruct files: %{},
+            pack_file: nil,
+            path: nil,
+            pack: %{},
+            name: nil
+
+  @type t() :: %__MODULE__{
+          files: %{String.t() => Path.t()},
+          pack_file: Path.t(),
+          path: Path.t(),
+          pack: map(),
+          name: String.t()
+        }
+
+  alias Pleroma.Emoji
+
+  @spec emoji_path() :: Path.t()
+  def emoji_path do
+    static = Pleroma.Config.get!([:instance, :static_dir])
+    Path.join(static, "emoji")
+  end
+
+  @spec create(String.t()) :: :ok | {:error, File.posix()} | {:error, :empty_values}
+  def create(name) when byte_size(name) > 0 do
+    dir = Path.join(emoji_path(), name)
+
+    with :ok <- File.mkdir(dir) do
+      %__MODULE__{
+        pack_file: Path.join(dir, "pack.json")
+      }
+      |> save_pack()
+    end
+  end
+
+  def create(_), do: {:error, :empty_values}
+
+  @spec show(String.t()) :: {:ok, t()} | {:loaded, nil} | {:error, :empty_values}
+  def show(name) when byte_size(name) > 0 do
+    with {_, %__MODULE__{} = pack} <- {:loaded, load_pack(name)},
+         {_, pack} <- validate_pack(pack) do
+      {:ok, pack}
+    end
+  end
+
+  def show(_), do: {:error, :empty_values}
+
+  @spec delete(String.t()) ::
+          {:ok, [binary()]} | {:error, File.posix(), binary()} | {:error, :empty_values}
+  def delete(name) when byte_size(name) > 0 do
+    emoji_path()
+    |> Path.join(name)
+    |> File.rm_rf()
+  end
+
+  def delete(_), do: {:error, :empty_values}
+
+  @spec add_file(String.t(), String.t(), Path.t(), Plug.Upload.t() | String.t()) ::
+          {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
+  def add_file(name, shortcode, filename, file)
+      when byte_size(name) > 0 and byte_size(shortcode) > 0 and byte_size(filename) > 0 do
+    with {_, nil} <- {:exists, Emoji.get(shortcode)},
+         {_, %__MODULE__{} = pack} <- {:loaded, load_pack(name)} do
+      file_path = Path.join(pack.path, filename)
+
+      create_subdirs(file_path)
+
+      case file do
+        %Plug.Upload{path: upload_path} ->
+          # Copy the uploaded file from the temporary directory
+          File.copy!(upload_path, file_path)
+
+        url when is_binary(url) ->
+          # Download and write the file
+          file_contents = Tesla.get!(url).body
+          File.write!(file_path, file_contents)
+      end
+
+      files = Map.put(pack.files, shortcode, filename)
+
+      updated_pack = %{pack | files: files}
+
+      case save_pack(updated_pack) do
+        :ok ->
+          Emoji.reload()
+          {:ok, updated_pack}
+
+        e ->
+          e
+      end
+    end
+  end
+
+  def add_file(_, _, _, _), do: {:error, :empty_values}
+
+  defp create_subdirs(file_path) do
+    if String.contains?(file_path, "/") do
+      file_path
+      |> Path.dirname()
+      |> File.mkdir_p!()
+    end
+  end
+
+  @spec delete_file(String.t(), String.t()) ::
+          {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
+  def delete_file(name, shortcode) when byte_size(name) > 0 and byte_size(shortcode) > 0 do
+    with {_, %__MODULE__{} = pack} <- {:loaded, load_pack(name)},
+         {_, {filename, files}} when not is_nil(filename) <-
+           {:exists, Map.pop(pack.files, shortcode)},
+         emoji <- Path.join(pack.path, filename),
+         {_, true} <- {:exists, File.exists?(emoji)} do
+      emoji_dir = Path.dirname(emoji)
+
+      File.rm!(emoji)
+
+      if String.contains?(filename, "/") and File.ls!(emoji_dir) == [] do
+        File.rmdir!(emoji_dir)
+      end
+
+      updated_pack = %{pack | files: files}
+
+      case save_pack(updated_pack) do
+        :ok ->
+          Emoji.reload()
+          {:ok, updated_pack}
+
+        e ->
+          e
+      end
+    end
+  end
+
+  def delete_file(_, _), do: {:error, :empty_values}
+
+  @spec update_file(String.t(), String.t(), String.t(), String.t(), boolean()) ::
+          {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
+  def update_file(name, shortcode, new_shortcode, new_filename, force)
+      when byte_size(name) > 0 and byte_size(shortcode) > 0 and byte_size(new_shortcode) > 0 and
+             byte_size(new_filename) > 0 do
+    with {_, %__MODULE__{} = pack} <- {:loaded, load_pack(name)},
+         {_, {filename, files}} when not is_nil(filename) <-
+           {:exists, Map.pop(pack.files, shortcode)},
+         {_, true} <- {:not_used, force or is_nil(Emoji.get(new_shortcode))} do
+      old_path = Path.join(pack.path, filename)
+      old_dir = Path.dirname(old_path)
+      new_path = Path.join(pack.path, new_filename)
+
+      create_subdirs(new_path)
+
+      :ok = File.rename(old_path, new_path)
+
+      if String.contains?(filename, "/") and File.ls!(old_dir) == [] do
+        File.rmdir!(old_dir)
+      end
+
+      files = Map.put(files, new_shortcode, new_filename)
+
+      updated_pack = %{pack | files: files}
+
+      case save_pack(updated_pack) do
+        :ok ->
+          Emoji.reload()
+          {:ok, updated_pack}
+
+        e ->
+          e
+      end
+    end
+  end
+
+  def update_file(_, _, _, _, _), do: {:error, :empty_values}
+
+  @spec import_from_filesystem() :: {:ok, [String.t()]} | {:error, atom()}
+  def import_from_filesystem do
+    emoji_path = emoji_path()
+
+    with {:ok, %{access: :read_write}} <- File.stat(emoji_path),
+         {:ok, results} <- File.ls(emoji_path) do
+      names =
+        results
+        |> Enum.map(&Path.join(emoji_path, &1))
+        |> Enum.reject(fn path ->
+          File.dir?(path) and File.exists?(Path.join(path, "pack.json"))
+        end)
+        |> Enum.map(&write_pack_contents/1)
+        |> Enum.filter(& &1)
+
+      {:ok, names}
+    else
+      {:ok, %{access: _}} -> {:error, :no_read_write}
+      e -> e
+    end
+  end
+
+  defp write_pack_contents(path) do
+    pack = %__MODULE__{
+      files: files_from_path(path),
+      path: path,
+      pack_file: Path.join(path, "pack.json")
+    }
+
+    case save_pack(pack) do
+      :ok -> Path.basename(path)
+      _ -> nil
+    end
+  end
+
+  defp files_from_path(path) do
+    txt_path = Path.join(path, "emoji.txt")
+
+    if File.exists?(txt_path) do
+      # There's an emoji.txt file, it's likely from a pack installed by the pack manager.
+      # Make a pack.json file from the contents of that emoji.txt file
+
+      # FIXME: Copy-pasted from Pleroma.Emoji/load_from_file_stream/2
+
+      # Create a map of shortcodes to filenames from emoji.txt
+      File.read!(txt_path)
+      |> String.split("\n")
+      |> Enum.map(&String.trim/1)
+      |> Enum.map(fn line ->
+        case String.split(line, ~r/,\s*/) do
+          # This matches both strings with and without tags
+          # and we don't care about tags here
+          [name, file | _] ->
+            file_dir_name = Path.dirname(file)
+
+            file =
+              if String.ends_with?(path, file_dir_name) do
+                Path.basename(file)
+              else
+                file
+              end
+
+            {name, file}
+
+          _ ->
+            nil
+        end
+      end)
+      |> Enum.filter(& &1)
+      |> Enum.into(%{})
+    else
+      # If there's no emoji.txt, assume all files
+      # that are of certain extensions from the config are emojis and import them all
+      pack_extensions = Pleroma.Config.get!([:emoji, :pack_extensions])
+      Emoji.Loader.make_shortcode_to_file_map(path, pack_extensions)
+    end
+  end
+
+  @spec list_remote(String.t()) :: {:ok, map()}
+  def list_remote(url) do
+    uri =
+      url
+      |> String.trim()
+      |> URI.parse()
+
+    with {_, true} <- {:shareable, shareable_packs_available?(uri)} do
+      packs =
+        uri
+        |> URI.merge("/api/pleroma/emoji/packs")
+        |> to_string()
+        |> Tesla.get!()
+        |> Map.get(:body)
+        |> Jason.decode!()
+
+      {:ok, packs}
+    end
+  end
+
+  @spec list_local() :: {:ok, map()}
+  def list_local do
+    emoji_path = emoji_path()
+
+    # Create the directory first if it does not exist. This is probably the first request made
+    # with the API so it should be sufficient
+    with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(emoji_path)},
+         {:ls, {:ok, results}} <- {:ls, File.ls(emoji_path)} do
+      packs =
+        results
+        |> Enum.map(&load_pack/1)
+        |> Enum.filter(& &1)
+        |> Enum.map(&validate_pack/1)
+        |> Map.new()
+
+      {:ok, packs}
+    end
+  end
+
+  defp validate_pack(pack) do
+    if downloadable?(pack) do
+      archive = fetch_archive(pack)
+      archive_sha = :crypto.hash(:sha256, archive) |> Base.encode16()
+
+      info =
+        pack.pack
+        |> Map.put("can-download", true)
+        |> Map.put("download-sha256", archive_sha)
+
+      {pack.name, Map.put(pack, :pack, info)}
+    else
+      info = Map.put(pack.pack, "can-download", false)
+      {pack.name, Map.put(pack, :pack, info)}
+    end
+  end
+
+  defp downloadable?(pack) do
+    # If the pack is set as shared, check if it can be downloaded
+    # That means that when asked, the pack can be packed and sent to the remote
+    # Otherwise, they'd have to download it from external-src
+    pack.pack["share-files"] &&
+      Enum.all?(pack.files, fn {_, file} ->
+        File.exists?(Path.join(pack.path, file))
+      end)
+  end
+
+  @spec get_archive(String.t()) :: {:ok, binary()}
+  def get_archive(name) do
+    with {_, %__MODULE__{} = pack} <- {:exists?, load_pack(name)},
+         {_, true} <- {:can_download?, downloadable?(pack)} do
+      {:ok, fetch_archive(pack)}
+    end
+  end
+
+  defp fetch_archive(pack) do
+    hash = :crypto.hash(:md5, File.read!(pack.pack_file))
+
+    case Cachex.get!(:emoji_packs_cache, pack.name) do
+      %{hash: ^hash, pack_data: archive} ->
+        archive
+
+      _ ->
+        create_archive_and_cache(pack, hash)
+    end
+  end
+
+  defp create_archive_and_cache(pack, hash) do
+    files = ['pack.json' | Enum.map(pack.files, fn {_, file} -> to_charlist(file) end)]
+
+    {:ok, {_, result}} =
+      :zip.zip('#{pack.name}.zip', files, [:memory, cwd: to_charlist(pack.path)])
+
+    ttl_per_file = Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file])
+    overall_ttl = :timer.seconds(ttl_per_file * Enum.count(files))
+
+    Cachex.put!(
+      :emoji_packs_cache,
+      pack.name,
+      # if pack.json MD5 changes, the cache is not valid anymore
+      %{hash: hash, pack_data: result},
+      # Add a minute to cache time for every file in the pack
+      ttl: overall_ttl
+    )
+
+    result
+  end
+
+  @spec download(String.t(), String.t(), String.t()) :: :ok
+  def download(name, url, as) do
+    uri =
+      url
+      |> String.trim()
+      |> URI.parse()
+
+    with {_, true} <- {:shareable, shareable_packs_available?(uri)} do
+      remote_pack =
+        uri
+        |> URI.merge("/api/pleroma/emoji/packs/#{name}")
+        |> to_string()
+        |> Tesla.get!()
+        |> Map.get(:body)
+        |> Jason.decode!()
+
+      result =
+        case remote_pack["pack"] do
+          %{"share-files" => true, "can-download" => true, "download-sha256" => sha} ->
+            {:ok,
+             %{
+               sha: sha,
+               url: URI.merge(uri, "/api/pleroma/emoji/packs/#{name}/archive") |> to_string()
+             }}
+
+          %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) ->
+            {:ok,
+             %{
+               sha: sha,
+               url: src,
+               fallback: true
+             }}
+
+          _ ->
+            {:error,
+             "The pack was not set as shared and there is no fallback src to download from"}
+        end
+
+      with {:ok, %{sha: sha, url: url} = pinfo} <- result,
+           %{body: archive} <- Tesla.get!(url),
+           {_, true} <- {:checksum, Base.decode16!(sha) == :crypto.hash(:sha256, archive)} do
+        local_name = as || name
+
+        path = Path.join(emoji_path(), local_name)
+
+        pack = %__MODULE__{
+          name: local_name,
+          path: path,
+          files: remote_pack["files"],
+          pack_file: Path.join(path, "pack.json")
+        }
+
+        File.mkdir_p!(pack.path)
+
+        files = Enum.map(remote_pack["files"], fn {_, path} -> to_charlist(path) end)
+        # Fallback cannot contain a pack.json file
+        files = if pinfo[:fallback], do: files, else: ['pack.json' | files]
+
+        {:ok, _} = :zip.unzip(archive, cwd: to_charlist(pack.path), file_list: files)
+
+        # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256
+        # in it to depend on itself
+        if pinfo[:fallback] do
+          save_pack(pack)
+        end
+
+        :ok
+      end
+    end
+  end
+
+  defp save_pack(pack), do: File.write(pack.pack_file, Jason.encode!(pack, pretty: true))
+
+  @spec save_metadata(map(), t()) :: {:ok, t()} | {:error, File.posix()}
+  def save_metadata(metadata, %__MODULE__{} = pack) do
+    pack = Map.put(pack, :pack, metadata)
+
+    with :ok <- save_pack(pack) do
+      {:ok, pack}
+    end
+  end
+
+  @spec update_metadata(String.t(), map()) :: {:ok, t()} | {:error, File.posix()}
+  def update_metadata(name, data) do
+    pack = load_pack(name)
+
+    fb_sha_changed? =
+      not is_nil(data["fallback-src"]) and data["fallback-src"] != pack.pack["fallback-src"]
+
+    with {_, true} <- {:update?, fb_sha_changed?},
+         {:ok, %{body: zip}} <- Tesla.get(data["fallback-src"]),
+         {:ok, f_list} <- :zip.unzip(zip, [:memory]),
+         {_, true} <- {:has_all_files?, has_all_files?(pack.files, f_list)} do
+      fallback_sha = :crypto.hash(:sha256, zip) |> Base.encode16()
+
+      data
+      |> Map.put("fallback-src-sha256", fallback_sha)
+      |> save_metadata(pack)
+    else
+      {:update?, _} -> save_metadata(data, pack)
+      e -> e
+    end
+  end
+
+  # Check if all files from the pack.json are in the archive
+  defp has_all_files?(files, f_list) do
+    Enum.all?(files, fn {_, from_manifest} ->
+      List.keyfind(f_list, to_charlist(from_manifest), 0)
+    end)
+  end
+
+  @spec load_pack(String.t()) :: t() | nil
+  def load_pack(name) do
+    pack_file = Path.join([emoji_path(), name, "pack.json"])
+
+    if File.exists?(pack_file) do
+      pack_file
+      |> File.read!()
+      |> from_json()
+      |> Map.put(:pack_file, pack_file)
+      |> Map.put(:path, Path.dirname(pack_file))
+      |> Map.put(:name, name)
+    end
+  end
+
+  defp from_json(json) do
+    map = Jason.decode!(json)
+
+    struct(__MODULE__, %{files: map["files"], pack: map["pack"]})
+  end
+
+  defp shareable_packs_available?(uri) do
+    uri
+    |> URI.merge("/.well-known/nodeinfo")
+    |> to_string()
+    |> Tesla.get!()
+    |> Map.get(:body)
+    |> Jason.decode!()
+    |> Map.get("links")
+    |> List.last()
+    |> Map.get("href")
+    # Get the actual nodeinfo address and fetch it
+    |> Tesla.get!()
+    |> Map.get(:body)
+    |> Jason.decode!()
+    |> get_in(["metadata", "features"])
+    |> Enum.member?("shareable_emoji_packs")
+  end
+end

lib/pleroma/filter.ex

@@ -89,11 +89,10 @@ def delete(%Pleroma.Filter{id: filter_key} = filter) when is_nil(filter_key) do
     |> Repo.delete()
   end
 
-  def update(%Pleroma.Filter{} = filter) do
+  def update(%Pleroma.Filter{} = filter, params) do
-    destination = Map.from_struct(filter)
+    filter
-
+    |> cast(params, [:phrase, :context, :hide, :expires_at, :whole_word])
-    Pleroma.Filter.get(filter.filter_id, %{id: filter.user_id})
+    |> validate_required([:phrase, :context])
-    |> cast(destination, [:phrase, :context, :hide, :expires_at, :whole_word])
     |> Repo.update()
   end
 end

lib/pleroma/following_relationship.ex

@@ -8,12 +8,14 @@ defmodule Pleroma.FollowingRelationship do
   import Ecto.Changeset
   import Ecto.Query
 
+  alias Ecto.Changeset
   alias FlakeId.Ecto.CompatType
+  alias Pleroma.FollowingRelationship.State
   alias Pleroma.Repo
   alias Pleroma.User
 
   schema "following_relationships" do
-    field(:state, :string, default: "accept")
+    field(:state, State, default: :follow_pending)
 
     belongs_to(:follower, User, type: CompatType)
     belongs_to(:following, User, type: CompatType)
@@ -21,12 +23,29 @@ defmodule Pleroma.FollowingRelationship do
     timestamps()
   end
 
+  @doc "Returns underlying integer code for state atom"
+  def state_int_code(state_atom), do: State.__enum_map__() |> Keyword.fetch!(state_atom)
+
+  def accept_state_code, do: state_int_code(:follow_accept)
+
   def changeset(%__MODULE__{} = following_relationship, attrs) do
     following_relationship
     |> cast(attrs, [:state])
     |> put_assoc(:follower, attrs.follower)
     |> put_assoc(:following, attrs.following)
     |> validate_required([:state, :follower, :following])
+    |> unique_constraint(:follower_id,
+      name: :following_relationships_follower_id_following_id_index
+    )
+    |> validate_not_self_relationship()
+  end
+
+  def state_to_enum(state) when state in ["pending", "accept", "reject"] do
+    String.to_existing_atom("follow_#{state}")
+  end
+
+  def state_to_enum(state) do
+    raise "State is not convertible to Pleroma.FollowingRelationship.State: #{state}"
   end
 
   def get(%User{} = follower, %User{} = following) do
@@ -35,7 +54,7 @@ def get(%User{} = follower, %User{} = following) do
     |> Repo.one()
   end
 
-  def update(follower, following, "reject"), do: unfollow(follower, following)
+  def update(follower, following, :follow_reject), do: unfollow(follower, following)
 
   def update(%User{} = follower, %User{} = following, state) do
     case get(follower, following) do
@@ -50,7 +69,7 @@ def update(%User{} = follower, %User{} = following, state) do
     end
   end
 
-  def follow(%User{} = follower, %User{} = following, state \\ "accept") do
+  def follow(%User{} = follower, %User{} = following, state \\ :follow_accept) do
     %__MODULE__{}
     |> changeset(%{follower: follower, following: following, state: state})
     |> Repo.insert(on_conflict: :nothing)
@@ -69,6 +88,29 @@ def follower_count(%User{} = user) do
     |> Repo.aggregate(:count, :id)
   end
 
+  def followers_query(%User{} = user) do
+    __MODULE__
+    |> join(:inner, [r], u in User, on: r.follower_id == u.id)
+    |> where([r], r.following_id == ^user.id)
+    |> where([r], r.state == ^:follow_accept)
+  end
+
+  def followers_ap_ids(%User{} = user, from_ap_ids \\ nil) do
+    query =
+      user
+      |> followers_query()
+      |> select([r, u], u.ap_id)
+
+    query =
+      if from_ap_ids do
+        where(query, [r, u], u.ap_id in ^from_ap_ids)
+      else
+        query
+      end
+
+    Repo.all(query)
+  end
+
   def following_count(%User{id: nil}), do: 0
 
   def following_count(%User{} = user) do
@@ -80,7 +122,7 @@ def following_count(%User{} = user) do
   def get_follow_requests(%User{id: id}) do
     __MODULE__
     |> join(:inner, [r], f in assoc(r, :follower))
-    |> where([r], r.state == "pending")
+    |> where([r], r.state == ^:follow_pending)
     |> where([r], r.following_id == ^id)
     |> select([r, f], f)
     |> Repo.all()
@@ -88,16 +130,20 @@ def get_follow_requests(%User{id: id}) do
 
   def following?(%User{id: follower_id}, %User{id: followed_id}) do
     __MODULE__
-    |> where(follower_id: ^follower_id, following_id: ^followed_id, state: "accept")
+    |> where(follower_id: ^follower_id, following_id: ^followed_id, state: ^:follow_accept)
     |> Repo.exists?()
   end
 
+  def following_query(%User{} = user) do
+    __MODULE__
+    |> join(:inner, [r], u in User, on: r.following_id == u.id)
+    |> where([r], r.follower_id == ^user.id)
+    |> where([r], r.state == ^:follow_accept)
+  end
+
   def following(%User{} = user) do
     following =
-      __MODULE__
+      following_query(user)
-      |> join(:inner, [r], u in User, on: r.following_id == u.id)
-      |> where([r], r.follower_id == ^user.id)
-      |> where([r], r.state == "accept")
       |> select([r, u], u.follower_address)
       |> Repo.all()
 
@@ -129,4 +175,82 @@ def move_following(origin, target) do
         move_following(origin, target)
     end
   end
+
+  def all_between_user_sets(
+        source_users,
+        target_users
+      )
+      when is_list(source_users) and is_list(target_users) do
+    source_user_ids = User.binary_id(source_users)
+    target_user_ids = User.binary_id(target_users)
+
+    __MODULE__
+    |> where(
+      fragment(
+        "(follower_id = ANY(?) AND following_id = ANY(?)) OR \
+        (follower_id = ANY(?) AND following_id = ANY(?))",
+        ^source_user_ids,
+        ^target_user_ids,
+        ^target_user_ids,
+        ^source_user_ids
+      )
+    )
+    |> Repo.all()
+  end
+
+  def find(following_relationships, follower, following) do
+    Enum.find(following_relationships, fn
+      fr -> fr.follower_id == follower.id and fr.following_id == following.id
+    end)
+  end
+
+  @doc """
+  For a query with joined activity,
+  keeps rows where activity's actor is followed by user -or- is NOT domain-blocked by user.
+  """
+  def keep_following_or_not_domain_blocked(query, user) do
+    where(
+      query,
+      [_, activity],
+      fragment(
+        # "(actor's domain NOT in domain_blocks) OR (actor IS in followed AP IDs)"
+        """
+        NOT (substring(? from '.*://([^/]*)') = ANY(?)) OR
+          ? = ANY(SELECT ap_id FROM users AS u INNER JOIN following_relationships AS fr
+            ON u.id = fr.following_id WHERE fr.follower_id = ? AND fr.state = ?)
+        """,
+        activity.actor,
+        ^user.domain_blocks,
+        activity.actor,
+        ^User.binary_id(user.id),
+        ^accept_state_code()
+      )
+    )
+  end
+
+  defp validate_not_self_relationship(%Changeset{} = changeset) do
+    changeset
+    |> validate_follower_id_following_id_inequality()
+    |> validate_following_id_follower_id_inequality()
+  end
+
+  defp validate_follower_id_following_id_inequality(%Changeset{} = changeset) do
+    validate_change(changeset, :follower_id, fn _, follower_id ->
+      if follower_id == get_field(changeset, :following_id) do
+        [source_id: "can't be equal to following_id"]
+      else
+        []
+      end
+    end)
+  end
+
+  defp validate_following_id_follower_id_inequality(%Changeset{} = changeset) do
+    validate_change(changeset, :following_id, fn _, following_id ->
+      if following_id == get_field(changeset, :follower_id) do
+        [target_id: "can't be equal to follower_id"]
+      else
+        []
+      end
+    end)
+  end
 end

lib/pleroma/formatter.ex

@@ -31,13 +31,23 @@ def escape_mention_handler("@" <> nickname = mention, buffer, _, _) do
   def mention_handler("@" <> nickname, buffer, opts, acc) do
     case User.get_cached_by_nickname(nickname) do
       %User{id: id} = user ->
-        ap_id = get_ap_id(user)
+        user_url = user.uri || user.ap_id
         nickname_text = get_nickname_text(nickname, opts)
 
         link =
-          ~s(<span class="h-card"><a data-user="#{id}" class="u-url mention" href="#{ap_id}" rel="ugc">@<span>#{
+          Phoenix.HTML.Tag.content_tag(
-            nickname_text
+            :span,
-          }</span></a></span>)
+            Phoenix.HTML.Tag.content_tag(
+              :a,
+              ["@", Phoenix.HTML.Tag.content_tag(:span, nickname_text)],
+              "data-user": id,
+              class: "u-url mention",
+              href: user_url,
+              rel: "ugc"
+            ),
+            class: "h-card"
+          )
+          |> Phoenix.HTML.safe_to_string()
 
         {link, %{acc | mentions: MapSet.put(acc.mentions, {"@" <> nickname, user})}}
 
@@ -49,7 +59,15 @@ def mention_handler("@" <> nickname, buffer, opts, acc) do
   def hashtag_handler("#" <> tag = tag_text, _buffer, _opts, acc) do
     tag = String.downcase(tag)
     url = "#{Pleroma.Web.base_url()}/tag/#{tag}"
-    link = ~s(<a class="hashtag" data-tag="#{tag}" href="#{url}" rel="tag ugc">#{tag_text}</a>)
+
+    link =
+      Phoenix.HTML.Tag.content_tag(:a, tag_text,
+        class: "hashtag",
+        "data-tag": tag,
+        href: url,
+        rel: "tag ugc"
+      )
+      |> Phoenix.HTML.safe_to_string()
 
     {link, %{acc | tags: MapSet.put(acc.tags, {tag_text, tag})}}
   end
@@ -128,9 +146,6 @@ def truncate(text, max_length \\ 200, omission \\ "...") do
     end
   end
 
-  defp get_ap_id(%User{source_data: %{"url" => url}}) when is_binary(url), do: url
-  defp get_ap_id(%User{ap_id: ap_id}), do: ap_id
-
   defp get_nickname_text(nickname, %{mentions_format: :full}), do: User.full_nickname(nickname)
   defp get_nickname_text(nickname, _), do: User.local_nickname(nickname)
 end

lib/pleroma/gun/api.ex

@@ -0,0 +1,45 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Gun.API do
+  @behaviour Pleroma.Gun
+
+  alias Pleroma.Gun
+
+  @gun_keys [
+    :connect_timeout,
+    :http_opts,
+    :http2_opts,
+    :protocols,
+    :retry,
+    :retry_timeout,
+    :trace,
+    :transport,
+    :tls_opts,
+    :tcp_opts,
+    :socks_opts,
+    :ws_opts
+  ]
+
+  @impl Gun
+  def open(host, port, opts \\ %{}), do: :gun.open(host, port, Map.take(opts, @gun_keys))
+
+  @impl Gun
+  defdelegate info(pid), to: :gun
+
+  @impl Gun
+  defdelegate close(pid), to: :gun
+
+  @impl Gun
+  defdelegate await_up(pid, timeout \\ 5_000), to: :gun
+
+  @impl Gun
+  defdelegate connect(pid, opts), to: :gun
+
+  @impl Gun
+  defdelegate await(pid, ref), to: :gun
+
+  @impl Gun
+  defdelegate set_owner(pid, owner), to: :gun
+end

lib/pleroma/gun/conn.ex

@@ -0,0 +1,198 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Gun.Conn do
+  @moduledoc """
+  Struct for gun connection data
+  """
+  alias Pleroma.Gun
+  alias Pleroma.Pool.Connections
+
+  require Logger
+
+  @type gun_state :: :up | :down
+  @type conn_state :: :active | :idle
+
+  @type t :: %__MODULE__{
+          conn: pid(),
+          gun_state: gun_state(),
+          conn_state: conn_state(),
+          used_by: [pid()],
+          last_reference: pos_integer(),
+          crf: float(),
+          retries: pos_integer()
+        }
+
+  defstruct conn: nil,
+            gun_state: :open,
+            conn_state: :init,
+            used_by: [],
+            last_reference: 0,
+            crf: 1,
+            retries: 0
+
+  @spec open(String.t() | URI.t(), atom(), keyword()) :: :ok | nil
+  def open(url, name, opts \\ [])
+  def open(url, name, opts) when is_binary(url), do: open(URI.parse(url), name, opts)
+
+  def open(%URI{} = uri, name, opts) do
+    pool_opts = Pleroma.Config.get([:connections_pool], [])
+
+    opts =
+      opts
+      |> Enum.into(%{})
+      |> Map.put_new(:retry, pool_opts[:retry] || 1)
+      |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000)
+      |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000)
+      |> maybe_add_tls_opts(uri)
+
+    key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
+
+    max_connections = pool_opts[:max_connections] || 250
+
+    conn_pid =
+      if Connections.count(name) < max_connections do
+        do_open(uri, opts)
+      else
+        close_least_used_and_do_open(name, uri, opts)
+      end
+
+    if is_pid(conn_pid) do
+      conn = %Pleroma.Gun.Conn{
+        conn: conn_pid,
+        gun_state: :up,
+        conn_state: :active,
+        last_reference: :os.system_time(:second)
+      }
+
+      :ok = Gun.set_owner(conn_pid, Process.whereis(name))
+      Connections.add_conn(name, key, conn)
+    end
+  end
+
+  defp maybe_add_tls_opts(opts, %URI{scheme: "http"}), do: opts
+
+  defp maybe_add_tls_opts(opts, %URI{scheme: "https", host: host}) do
+    tls_opts = [
+      verify: :verify_peer,
+      cacertfile: CAStore.file_path(),
+      depth: 20,
+      reuse_sessions: false,
+      verify_fun:
+        {&:ssl_verify_hostname.verify_fun/3,
+         [check_hostname: Pleroma.HTTP.Connection.format_host(host)]}
+    ]
+
+    tls_opts =
+      if Keyword.keyword?(opts[:tls_opts]) do
+        Keyword.merge(tls_opts, opts[:tls_opts])
+      else
+        tls_opts
+      end
+
+    Map.put(opts, :tls_opts, tls_opts)
+  end
+
+  defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do
+    connect_opts =
+      uri
+      |> destination_opts()
+      |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
+
+    with open_opts <- Map.delete(opts, :tls_opts),
+         {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
+         {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]),
+         stream <- Gun.connect(conn, connect_opts),
+         {:response, :fin, 200, _} <- Gun.await(conn, stream) do
+      conn
+    else
+      error ->
+        Logger.warn(
+          "Opening proxied connection to #{compose_uri_log(uri)} failed with error #{
+            inspect(error)
+          }"
+        )
+
+        error
+    end
+  end
+
+  defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do
+    version =
+      proxy_type
+      |> to_string()
+      |> String.last()
+      |> case do
+        "4" -> 4
+        _ -> 5
+      end
+
+    socks_opts =
+      uri
+      |> destination_opts()
+      |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
+      |> Map.put(:version, version)
+
+    opts =
+      opts
+      |> Map.put(:protocols, [:socks])
+      |> Map.put(:socks_opts, socks_opts)
+
+    with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
+         {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
+      conn
+    else
+      error ->
+        Logger.warn(
+          "Opening socks proxied connection to #{compose_uri_log(uri)} failed with error #{
+            inspect(error)
+          }"
+        )
+
+        error
+    end
+  end
+
+  defp do_open(%URI{host: host, port: port} = uri, opts) do
+    host = Pleroma.HTTP.Connection.parse_host(host)
+
+    with {:ok, conn} <- Gun.open(host, port, opts),
+         {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
+      conn
+    else
+      error ->
+        Logger.warn(
+          "Opening connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}"
+        )
+
+        error
+    end
+  end
+
+  defp destination_opts(%URI{host: host, port: port}) do
+    host = Pleroma.HTTP.Connection.parse_host(host)
+    %{host: host, port: port}
+  end
+
+  defp add_http2_opts(opts, "https", tls_opts) do
+    Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts})
+  end
+
+  defp add_http2_opts(opts, _, _), do: opts
+
+  defp close_least_used_and_do_open(name, uri, opts) do
+    with [{key, conn} | _conns] <- Connections.get_unused_conns(name),
+         :ok <- Gun.close(conn.conn) do
+      Connections.remove_conn(name, key)
+
+      do_open(uri, opts)
+    else
+      [] -> {:error, :pool_overflowed}
+    end
+  end
+
+  def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do
+    "#{scheme}://#{host}#{path}"
+  end
+end

lib/pleroma/gun/gun.ex

@@ -0,0 +1,31 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Gun do
+  @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()}
+  @callback info(pid()) :: map()
+  @callback close(pid()) :: :ok
+  @callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()}
+  @callback connect(pid(), map()) :: reference()
+  @callback await(pid(), reference()) :: {:response, :fin, 200, []}
+  @callback set_owner(pid(), pid()) :: :ok
+
+  @api Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API)
+
+  defp api, do: @api
+
+  def open(host, port, opts), do: api().open(host, port, opts)
+
+  def info(pid), do: api().info(pid)
+
+  def close(pid), do: api().close(pid)
+
+  def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout)
+
+  def connect(pid, opts), do: api().connect(pid, opts)
+
+  def await(pid, ref), do: api().await(pid, ref)
+
+  def set_owner(pid, owner), do: api().set_owner(pid, owner)
+end

lib/pleroma/healthcheck.ex

@@ -29,7 +29,7 @@ defmodule Pleroma.Healthcheck do
   @spec system_info() :: t()
   def system_info do
     %Healthcheck{
-      memory_used: Float.round(:erlang.memory(:total) / 1024 / 1024, 2)
+      memory_used: Float.round(:recon_alloc.memory(:allocated) / 1024 / 1024, 2)
     }
     |> assign_db_info()
     |> assign_job_queue_stats()

lib/pleroma/http/adapter_helper.ex

@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.AdapterHelper do
+  alias Pleroma.HTTP.Connection
+
+  @type proxy ::
+          {Connection.host(), pos_integer()}
+          | {Connection.proxy_type(), Connection.host(), pos_integer()}
+
+  @callback options(keyword(), URI.t()) :: keyword()
+  @callback after_request(keyword()) :: :ok
+
+  @spec options(keyword(), URI.t()) :: keyword()
+  def options(opts, _uri) do
+    proxy = Pleroma.Config.get([:http, :proxy_url], nil)
+    maybe_add_proxy(opts, format_proxy(proxy))
+  end
+
+  @spec maybe_get_conn(URI.t(), keyword()) :: keyword()
+  def maybe_get_conn(_uri, opts), do: opts
+
+  @spec after_request(keyword()) :: :ok
+  def after_request(_opts), do: :ok
+
+  @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil
+  def format_proxy(nil), do: nil
+
+  def format_proxy(proxy_url) do
+    case Connection.parse_proxy(proxy_url) do
+      {:ok, host, port} -> {host, port}
+      {:ok, type, host, port} -> {type, host, port}
+      _ -> nil
+    end
+  end
+
+  @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword()
+  def maybe_add_proxy(opts, nil), do: opts
+  def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy)
+end

lib/pleroma/http/adapter_helper/gun.ex

@@ -0,0 +1,77 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.AdapterHelper.Gun do
+  @behaviour Pleroma.HTTP.AdapterHelper
+
+  alias Pleroma.HTTP.AdapterHelper
+  alias Pleroma.Pool.Connections
+
+  require Logger
+
+  @defaults [
+    connect_timeout: 5_000,
+    domain_lookup_timeout: 5_000,
+    tls_handshake_timeout: 5_000,
+    retry: 1,
+    retry_timeout: 1000,
+    await_up_timeout: 5_000
+  ]
+
+  @spec options(keyword(), URI.t()) :: keyword()
+  def options(incoming_opts \\ [], %URI{} = uri) do
+    proxy =
+      Pleroma.Config.get([:http, :proxy_url])
+      |> AdapterHelper.format_proxy()
+
+    config_opts = Pleroma.Config.get([:http, :adapter], [])
+
+    @defaults
+    |> Keyword.merge(config_opts)
+    |> add_scheme_opts(uri)
+    |> AdapterHelper.maybe_add_proxy(proxy)
+    |> maybe_get_conn(uri, incoming_opts)
+  end
+
+  @spec after_request(keyword()) :: :ok
+  def after_request(opts) do
+    if opts[:conn] && opts[:body_as] != :chunks do
+      Connections.checkout(opts[:conn], self(), :gun_connections)
+    end
+
+    :ok
+  end
+
+  defp add_scheme_opts(opts, %{scheme: "http"}), do: opts
+
+  defp add_scheme_opts(opts, %{scheme: "https"}) do
+    opts
+    |> Keyword.put(:certificates_verification, true)
+    |> Keyword.put(:tls_opts, log_level: :warning)
+  end
+
+  defp maybe_get_conn(adapter_opts, uri, incoming_opts) do
+    {receive_conn?, opts} =
+      adapter_opts
+      |> Keyword.merge(incoming_opts)
+      |> Keyword.pop(:receive_conn, true)
+
+    if Connections.alive?(:gun_connections) and receive_conn? do
+      checkin_conn(uri, opts)
+    else
+      opts
+    end
+  end
+
+  defp checkin_conn(uri, opts) do
+    case Connections.checkin(uri, :gun_connections) do
+      nil ->
+        Task.start(Pleroma.Gun.Conn, :open, [uri, :gun_connections, opts])
+        opts
+
+      conn when is_pid(conn) ->
+        Keyword.merge(opts, conn: conn, close_conn: false)
+    end
+  end
+end

lib/pleroma/http/adapter_helper/hackney.ex

@@ -0,0 +1,43 @@
+defmodule Pleroma.HTTP.AdapterHelper.Hackney do
+  @behaviour Pleroma.HTTP.AdapterHelper
+
+  @defaults [
+    connect_timeout: 10_000,
+    recv_timeout: 20_000,
+    follow_redirect: true,
+    force_redirect: true,
+    pool: :federation
+  ]
+
+  @spec options(keyword(), URI.t()) :: keyword()
+  def options(connection_opts \\ [], %URI{} = uri) do
+    proxy = Pleroma.Config.get([:http, :proxy_url])
+
+    config_opts = Pleroma.Config.get([:http, :adapter], [])
+
+    @defaults
+    |> Keyword.merge(config_opts)
+    |> Keyword.merge(connection_opts)
+    |> add_scheme_opts(uri)
+    |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
+  end
+
+  defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts
+
+  defp add_scheme_opts(opts, %URI{scheme: "https", host: host}) do
+    ssl_opts = [
+      ssl_options: [
+        # Workaround for remote server certificate chain issues
+        partial_chain: &:hackney_connect.partial_chain/1,
+
+        # We don't support TLS v1.3 yet
+        versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"],
+        server_name_indication: to_charlist(host)
+      ]
+    ]
+
+    Keyword.merge(opts, ssl_opts)
+  end
+
+  def after_request(_), do: :ok
+end

lib/pleroma/http/connection.ex

@@ -4,40 +4,121 @@
 
 defmodule Pleroma.HTTP.Connection do
   @moduledoc """
-  Connection for http-requests.
+  Configure Tesla.Client with default and customized adapter options.
   """
 
-  @hackney_options [
+  alias Pleroma.Config
-    connect_timeout: 10_000,
+  alias Pleroma.HTTP.AdapterHelper
-    recv_timeout: 20_000,
+
-    follow_redirect: true,
+  require Logger
-    force_redirect: true,
+
-    pool: :federation
+  @defaults [pool: :federation]
-  ]
+
-  @adapter Application.get_env(:tesla, :adapter)
+  @type ip_address :: ipv4_address() | ipv6_address()
+  @type ipv4_address :: {0..255, 0..255, 0..255, 0..255}
+  @type ipv6_address ::
+          {0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535}
+  @type proxy_type() :: :socks4 | :socks5
+  @type host() :: charlist() | ip_address()
 
   @doc """
-  Configure a client connection
+  Merge default connection & adapter options with received ones.
-
-  # Returns
-
-  Tesla.Env.client
   """
-  @spec new(Keyword.t()) :: Tesla.Env.client()
+
-  def new(opts \\ []) do
+  @spec options(URI.t(), keyword()) :: keyword()
-    Tesla.client([], {@adapter, hackney_options(opts)})
+  def options(%URI{} = uri, opts \\ []) do
+    @defaults
+    |> pool_timeout()
+    |> Keyword.merge(opts)
+    |> adapter_helper().options(uri)
   end
 
-  # fetch Hackney options
+  defp pool_timeout(opts) do
-  #
+    {config_key, default} =
-  def hackney_options(opts) do
+      if adapter() == Tesla.Adapter.Gun do
-    options = Keyword.get(opts, :adapter, [])
+        {:pools, Config.get([:pools, :default, :timeout])}
-    adapter_options = Pleroma.Config.get([:http, :adapter], [])
+      else
-    proxy_url = Pleroma.Config.get([:http, :proxy_url], nil)
+        {:hackney_pools, 10_000}
+      end
 
-    @hackney_options
+    timeout = Config.get([config_key, opts[:pool], :timeout], default)
-    |> Keyword.merge(adapter_options)
+
-    |> Keyword.merge(options)
+    Keyword.merge(opts, timeout: timeout)
-    |> Keyword.merge(proxy: proxy_url)
+  end
+
+  @spec after_request(keyword()) :: :ok
+  def after_request(opts), do: adapter_helper().after_request(opts)
+
+  defp adapter, do: Application.get_env(:tesla, :adapter)
+
+  defp adapter_helper do
+    case adapter() do
+      Tesla.Adapter.Gun -> AdapterHelper.Gun
+      Tesla.Adapter.Hackney -> AdapterHelper.Hackney
+      _ -> AdapterHelper
+    end
+  end
+
+  @spec parse_proxy(String.t() | tuple() | nil) ::
+          {:ok, host(), pos_integer()}
+          | {:ok, proxy_type(), host(), pos_integer()}
+          | {:error, atom()}
+          | nil
+
+  def parse_proxy(nil), do: nil
+
+  def parse_proxy(proxy) when is_binary(proxy) do
+    with [host, port] <- String.split(proxy, ":"),
+         {port, ""} <- Integer.parse(port) do
+      {:ok, parse_host(host), port}
+    else
+      {_, _} ->
+        Logger.warn("Parsing port failed #{inspect(proxy)}")
+        {:error, :invalid_proxy_port}
+
+      :error ->
+        Logger.warn("Parsing port failed #{inspect(proxy)}")
+        {:error, :invalid_proxy_port}
+
+      _ ->
+        Logger.warn("Parsing proxy failed #{inspect(proxy)}")
+        {:error, :invalid_proxy}
+    end
+  end
+
+  def parse_proxy(proxy) when is_tuple(proxy) do
+    with {type, host, port} <- proxy do
+      {:ok, type, parse_host(host), port}
+    else
+      _ ->
+        Logger.warn("Parsing proxy failed #{inspect(proxy)}")
+        {:error, :invalid_proxy}
+    end
+  end
+
+  @spec parse_host(String.t() | atom() | charlist()) :: charlist() | ip_address()
+  def parse_host(host) when is_list(host), do: host
+  def parse_host(host) when is_atom(host), do: to_charlist(host)
+
+  def parse_host(host) when is_binary(host) do
+    host = to_charlist(host)
+
+    case :inet.parse_address(host) do
+      {:error, :einval} -> host
+      {:ok, ip} -> ip
+    end
+  end
+
+  @spec format_host(String.t()) :: charlist()
+  def format_host(host) do
+    host_charlist = to_charlist(host)
+
+    case :inet.parse_address(host_charlist) do
+      {:error, :einval} ->
+        :idna.encode(host_charlist)
+
+      {:ok, _ip} ->
+        host_charlist
+    end
   end
 end

lib/pleroma/http/http.ex

@@ -4,21 +4,47 @@
 
 defmodule Pleroma.HTTP do
   @moduledoc """
-
+    Wrapper for `Tesla.request/2`.
   """
 
   alias Pleroma.HTTP.Connection
+  alias Pleroma.HTTP.Request
   alias Pleroma.HTTP.RequestBuilder, as: Builder
+  alias Tesla.Client
+  alias Tesla.Env
+
+  require Logger
 
   @type t :: __MODULE__
 
   @doc """
-  Builds and perform http request.
+  Performs GET request.
+
+  See `Pleroma.HTTP.request/5`
+  """
+  @spec get(Request.url() | nil, Request.headers(), keyword()) ::
+          nil | {:ok, Env.t()} | {:error, any()}
+  def get(url, headers \\ [], options \\ [])
+  def get(nil, _, _), do: nil
+  def get(url, headers, options), do: request(:get, url, "", headers, options)
+
+  @doc """
+  Performs POST request.
+
+  See `Pleroma.HTTP.request/5`
+  """
+  @spec post(Request.url(), String.t(), Request.headers(), keyword()) ::
+          {:ok, Env.t()} | {:error, any()}
+  def post(url, body, headers \\ [], options \\ []),
+    do: request(:post, url, body, headers, options)
+
+  @doc """
+  Builds and performs http request.
 
   # Arguments:
   `method` - :get, :post, :put, :delete
-  `url`
+  `url` - full url
-  `body`
+  `body` - request body
   `headers` - a keyworld list of headers, e.g. `[{"content-type", "text/plain"}]`
   `options` - custom, per-request middleware or adapter options
 
@@ -26,61 +52,66 @@ defmodule Pleroma.HTTP do
   `{:ok, %Tesla.Env{}}` or `{:error, error}`
 
   """
-  def request(method, url, body \\ "", headers \\ [], options \\ []) do
+  @spec request(atom(), Request.url(), String.t(), Request.headers(), keyword()) ::
-    try do
+          {:ok, Env.t()} | {:error, any()}
-      options =
+  def request(method, url, body, headers, options) when is_binary(url) do
-        process_request_options(options)
-        |> process_sni_options(url)
-
-      params = Keyword.get(options, :params, [])
-
-      %{}
-      |> Builder.method(method)
-      |> Builder.headers(headers)
-      |> Builder.opts(options)
-      |> Builder.url(url)
-      |> Builder.add_param(:body, :body, body)
-      |> Builder.add_param(:query, :query, params)
-      |> Enum.into([])
-      |> (&Tesla.request(Connection.new(options), &1)).()
-    rescue
-      e ->
-        {:error, e}
-    catch
-      :exit, e ->
-        {:error, e}
-    end
-  end
-
-  defp process_sni_options(options, nil), do: options
-
-  defp process_sni_options(options, url) do
     uri = URI.parse(url)
-    host = uri.host |> to_charlist()
+    adapter_opts = Connection.options(uri, options[:adapter] || [])
+    options = put_in(options[:adapter], adapter_opts)
+    params = options[:params] || []
+    request = build_request(method, headers, options, url, body, params)
 
-    case uri.scheme do
+    adapter = Application.get_env(:tesla, :adapter)
-      "https" -> options ++ [ssl: [server_name_indication: host]]
+    client = Tesla.client([Tesla.Middleware.FollowRedirects], adapter)
-      _ -> options
+
-    end
+    pid = Process.whereis(adapter_opts[:pool])
+
+    pool_alive? =
+      if adapter == Tesla.Adapter.Gun && pid do
+        Process.alive?(pid)
+      else
+        false
+      end
+
+    request_opts =
+      adapter_opts
+      |> Enum.into(%{})
+      |> Map.put(:env, Pleroma.Config.get([:env]))
+      |> Map.put(:pool_alive?, pool_alive?)
+
+    response = request(client, request, request_opts)
+
+    Connection.after_request(adapter_opts)
+
+    response
   end
 
-  def process_request_options(options) do
+  @spec request(Client.t(), keyword(), map()) :: {:ok, Env.t()} | {:error, any()}
-    Keyword.merge(Pleroma.HTTP.Connection.hackney_options([]), options)
+  def request(%Client{} = client, request, %{env: :test}), do: request(client, request)
+
+  def request(%Client{} = client, request, %{body_as: :chunks}), do: request(client, request)
+
+  def request(%Client{} = client, request, %{pool_alive?: false}), do: request(client, request)
+
+  def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do
+    :poolboy.transaction(
+      pool,
+      &Pleroma.Pool.Request.execute(&1, client, request, timeout),
+      timeout
+    )
   end
 
-  @doc """
+  @spec request(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()}
-  Performs GET request.
+  def request(client, request), do: Tesla.request(client, request)
 
-  See `Pleroma.HTTP.request/5`
+  defp build_request(method, headers, options, url, body, params) do
-  """
+    Builder.new()
-  def get(url, headers \\ [], options \\ []),
+    |> Builder.method(method)
-    do: request(:get, url, "", headers, options)
+    |> Builder.headers(headers)
-
+    |> Builder.opts(options)
-  @doc """
+    |> Builder.url(url)
-  Performs POST request.
+    |> Builder.add_param(:body, :body, body)
-
+    |> Builder.add_param(:query, :query, params)
-  See `Pleroma.HTTP.request/5`
+    |> Builder.convert_to_keyword()
-  """
+  end
-  def post(url, body, headers \\ [], options \\ []),
-    do: request(:post, url, body, headers, options)
 end

lib/pleroma/http/request.ex

@@ -0,0 +1,23 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.Request do
+  @moduledoc """
+  Request struct.
+  """
+  defstruct method: :get, url: "", query: [], headers: [], body: "", opts: []
+
+  @type method :: :head | :get | :delete | :trace | :options | :post | :put | :patch
+  @type url :: String.t()
+  @type headers :: [{String.t(), String.t()}]
+
+  @type t :: %__MODULE__{
+          method: method(),
+          url: url(),
+          query: keyword(),
+          headers: headers(),
+          body: String.t(),
+          opts: keyword()
+        }
+end

lib/pleroma/http/request_builder.ex

@@ -7,136 +7,87 @@ defmodule Pleroma.HTTP.RequestBuilder do
   Helper functions for building Tesla requests
   """
 
+  alias Pleroma.HTTP.Request
+  alias Tesla.Multipart
+
   @doc """
-  Specify the request method when building a request
+  Creates new request
-
-  ## Parameters
-
-  - request (Map) - Collected request options
-  - m (atom) - Request method
-
-  ## Returns
-
-  Map
   """
-  @spec method(map(), atom) :: map()
+  @spec new(Request.t()) :: Request.t()
-  def method(request, m) do
+  def new(%Request{} = request \\ %Request{}), do: request
-    Map.put_new(request, :method, m)
-  end
 
   @doc """
   Specify the request method when building a request
-
-  ## Parameters
-
-  - request (Map) - Collected request options
-  - u (String) - Request URL
-
-  ## Returns
-
-  Map
   """
-  @spec url(map(), String.t()) :: map()
+  @spec method(Request.t(), Request.method()) :: Request.t()
-  def url(request, u) do
+  def method(request, m), do: %{request | method: m}
-    Map.put_new(request, :url, u)
+
-  end
+  @doc """
+  Specify the request method when building a request
+  """
+  @spec url(Request.t(), Request.url()) :: Request.t()
+  def url(request, u), do: %{request | url: u}
 
   @doc """
   Add headers to the request
   """
-  @spec headers(map(), list(tuple)) :: map()
+  @spec headers(Request.t(), Request.headers()) :: Request.t()
-  def headers(request, header_list) do
+  def headers(request, headers) do
-    header_list =
+    headers_list =
       if Pleroma.Config.get([:http, :send_user_agent]) do
-        header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}]
+        [{"user-agent", Pleroma.Application.user_agent()} | headers]
       else
-        header_list
+        headers
       end
 
-    Map.put_new(request, :headers, header_list)
+    %{request | headers: headers_list}
   end
 
   @doc """
   Add custom, per-request middleware or adapter options to the request
   """
-  @spec opts(map(), Keyword.t()) :: map()
+  @spec opts(Request.t(), keyword()) :: Request.t()
-  def opts(request, options) do
+  def opts(request, options), do: %{request | opts: options}
-    Map.put_new(request, :opts, options)
-  end
 
   @doc """
   Add optional parameters to the request
-
-  ## Parameters
-
-  - request (Map) - Collected request options
-  - definitions (Map) - Map of parameter name to parameter location.
-  - options (KeywordList) - The provided optional parameters
-
-  ## Returns
-
-  Map
   """
-  @spec add_optional_params(map(), %{optional(atom) => atom}, keyword()) :: map()
+  @spec add_param(Request.t(), atom(), atom(), any()) :: Request.t()
-  def add_optional_params(request, _, []), do: request
+  def add_param(request, :query, :query, values), do: %{request | query: values}
 
-  def add_optional_params(request, definitions, [{key, value} | tail]) do
+  def add_param(request, :body, :body, value), do: %{request | body: value}
-    case definitions do
-      %{^key => location} ->
-        request
-        |> add_param(location, key, value)
-        |> add_optional_params(definitions, tail)
-
-      _ ->
-        add_optional_params(request, definitions, tail)
-    end
-  end
-
-  @doc """
-  Add optional parameters to the request
-
-  ## Parameters
-
-  - request (Map) - Collected request options
-  - location (atom) - Where to put the parameter
-  - key (atom) - The name of the parameter
-  - value (any) - The value of the parameter
-
-  ## Returns
-
-  Map
-  """
-  @spec add_param(map(), atom, atom, any()) :: map()
-  def add_param(request, :query, :query, values), do: Map.put(request, :query, values)
-
-  def add_param(request, :body, :body, value), do: Map.put(request, :body, value)
 
   def add_param(request, :body, key, value) do
     request
-    |> Map.put_new_lazy(:body, &Tesla.Multipart.new/0)
+    |> Map.put(:body, Multipart.new())
     |> Map.update!(
       :body,
-      &Tesla.Multipart.add_field(
+      &Multipart.add_field(
         &1,
         key,
         Jason.encode!(value),
-        headers: [{:"Content-Type", "application/json"}]
+        headers: [{"content-type", "application/json"}]
       )
     )
   end
 
   def add_param(request, :file, name, path) do
     request
-    |> Map.put_new_lazy(:body, &Tesla.Multipart.new/0)
+    |> Map.put(:body, Multipart.new())
-    |> Map.update!(:body, &Tesla.Multipart.add_file(&1, path, name: name))
+    |> Map.update!(:body, &Multipart.add_file(&1, path, name: name))
   end
 
   def add_param(request, :form, name, value) do
-    request
+    Map.update(request, :body, %{name => value}, &Map.put(&1, name, value))
-    |> Map.update(:body, %{name => value}, &Map.put(&1, name, value))
   end
 
   def add_param(request, location, key, value) do
     Map.update(request, location, [{key, value}], &(&1 ++ [{key, value}]))
   end
+
+  def convert_to_keyword(request) do
+    request
+    |> Map.from_struct()
+    |> Enum.into([])
+  end
 end

lib/pleroma/marker.ex

@@ -9,24 +9,34 @@ defmodule Pleroma.Marker do
   import Ecto.Query
 
   alias Ecto.Multi
+  alias Pleroma.Notification
   alias Pleroma.Repo
   alias Pleroma.User
+  alias __MODULE__
 
   @timelines ["notifications"]
+  @type t :: %__MODULE__{}
 
   schema "markers" do
     field(:last_read_id, :string, default: "")
     field(:timeline, :string, default: "")
     field(:lock_version, :integer, default: 0)
+    field(:unread_count, :integer, default: 0, virtual: true)
 
     belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
     timestamps()
   end
 
+  @doc "Gets markers by user and timeline."
+  @spec get_markers(User.t(), list(String)) :: list(t())
   def get_markers(user, timelines \\ []) do
-    Repo.all(get_query(user, timelines))
+    user
+    |> get_query(timelines)
+    |> unread_count_query()
+    |> Repo.all()
   end
 
+  @spec upsert(User.t(), map()) :: {:ok | :error, any()}
   def upsert(%User{} = user, attrs) do
     attrs
     |> Map.take(@timelines)
@@ -45,6 +55,27 @@ def upsert(%User{} = user, attrs) do
     |> Repo.transaction()
   end
 
+  @spec multi_set_last_read_id(Multi.t(), User.t(), String.t()) :: Multi.t()
+  def multi_set_last_read_id(multi, %User{} = user, "notifications") do
+    multi
+    |> Multi.run(:counters, fn _repo, _changes ->
+      {:ok, %{last_read_id: Repo.one(Notification.last_read_query(user))}}
+    end)
+    |> Multi.insert(
+      :marker,
+      fn %{counters: attrs} ->
+        %Marker{timeline: "notifications", user_id: user.id}
+        |> struct(attrs)
+        |> Ecto.Changeset.change()
+      end,
+      returning: true,
+      on_conflict: {:replace, [:last_read_id]},
+      conflict_target: [:user_id, :timeline]
+    )
+  end
+
+  def multi_set_last_read_id(multi, _, _), do: multi
+
   defp get_marker(user, timeline) do
     case Repo.find_resource(get_query(user, timeline)) do
       {:ok, marker} -> %__MODULE__{marker | user: user}
@@ -71,4 +102,16 @@ defp get_query(user, timelines) do
     |> by_user_id(user.id)
     |> by_timeline(timelines)
   end
+
+  defp unread_count_query(query) do
+    from(
+      q in query,
+      left_join: n in "notifications",
+      on: n.user_id == q.user_id and n.seen == false,
+      group_by: [:id],
+      select_merge: %{
+        unread_count: fragment("count(?)", n.id)
+      }
+    )
+  end
 end

lib/pleroma/mfa.ex

@@ -0,0 +1,155 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.MFA do
+  @moduledoc """
+  The MFA context.
+  """
+
+  alias Pleroma.User
+
+  alias Pleroma.MFA.BackupCodes
+  alias Pleroma.MFA.Changeset
+  alias Pleroma.MFA.Settings
+  alias Pleroma.MFA.TOTP
+
+  @doc """
+  Returns MFA methods the user has enabled.
+
+  ## Examples
+
+    iex> Pleroma.MFA.supported_method(User)
+    "totp, u2f"
+  """
+  @spec supported_methods(User.t()) :: String.t()
+  def supported_methods(user) do
+    settings = fetch_settings(user)
+
+    Settings.mfa_methods()
+    |> Enum.reduce([], fn m, acc ->
+      if method_enabled?(m, settings) do
+        acc ++ [m]
+      else
+        acc
+      end
+    end)
+    |> Enum.join(",")
+  end
+
+  @doc "Checks that user enabled MFA"
+  def require?(user) do
+    fetch_settings(user).enabled
+  end
+
+  @doc """
+  Display MFA settings of user
+  """
+  def mfa_settings(user) do
+    settings = fetch_settings(user)
+
+    Settings.mfa_methods()
+    |> Enum.map(fn m -> [m, method_enabled?(m, settings)] end)
+    |> Enum.into(%{enabled: settings.enabled}, fn [a, b] -> {a, b} end)
+  end
+
+  @doc false
+  def fetch_settings(%User{} = user) do
+    user.multi_factor_authentication_settings || %Settings{}
+  end
+
+  @doc "clears backup codes"
+  def invalidate_backup_code(%User{} = user, hash_code) do
+    %{backup_codes: codes} = fetch_settings(user)
+
+    user
+    |> Changeset.cast_backup_codes(codes -- [hash_code])
+    |> User.update_and_set_cache()
+  end
+
+  @doc "generates backup codes"
+  @spec generate_backup_codes(User.t()) :: {:ok, list(binary)} | {:error, String.t()}
+  def generate_backup_codes(%User{} = user) do
+    with codes <- BackupCodes.generate(),
+         hashed_codes <- Enum.map(codes, &Pbkdf2.hash_pwd_salt/1),
+         changeset <- Changeset.cast_backup_codes(user, hashed_codes),
+         {:ok, _} <- User.update_and_set_cache(changeset) do
+      {:ok, codes}
+    else
+      {:error, msg} ->
+        %{error: msg}
+    end
+  end
+
+  @doc """
+  Generates secret key and set delivery_type to 'app' for TOTP method.
+  """
+  @spec setup_totp(User.t()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()}
+  def setup_totp(user) do
+    user
+    |> Changeset.setup_totp(%{secret: TOTP.generate_secret(), delivery_type: "app"})
+    |> User.update_and_set_cache()
+  end
+
+  @doc """
+  Confirms the TOTP method for user.
+
+  `attrs`:
+    `password` - current user password
+    `code` - TOTP token
+  """
+  @spec confirm_totp(User.t(), map()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t() | atom()}
+  def confirm_totp(%User{} = user, attrs) do
+    with settings <- user.multi_factor_authentication_settings.totp,
+         {:ok, :pass} <- TOTP.validate_token(settings.secret, attrs["code"]) do
+      user
+      |> Changeset.confirm_totp()
+      |> User.update_and_set_cache()
+    end
+  end
+
+  @doc """
+  Disables the TOTP method for user.
+
+  `attrs`:
+    `password` - current user password
+  """
+  @spec disable_totp(User.t()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()}
+  def disable_totp(%User{} = user) do
+    user
+    |> Changeset.disable_totp()
+    |> Changeset.disable()
+    |> User.update_and_set_cache()
+  end
+
+  @doc """
+  Force disables all MFA methods for user.
+  """
+  @spec disable(User.t()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()}
+  def disable(%User{} = user) do
+    user
+    |> Changeset.disable_totp()
+    |> Changeset.disable(true)
+    |> User.update_and_set_cache()
+  end
+
+  @doc """
+  Checks if the user has MFA method enabled.
+  """
+  def method_enabled?(method, settings) do
+    with {:ok, %{confirmed: true} = _} <- Map.fetch(settings, method) do
+      true
+    else
+      _ -> false
+    end
+  end
+
+  @doc """
+  Checks if the user has enabled at least one MFA method.
+  """
+  def enabled?(settings) do
+    Settings.mfa_methods()
+    |> Enum.map(fn m -> method_enabled?(m, settings) end)
+    |> Enum.any?()
+  end
+end

lib/pleroma/mfa/backup_codes.ex

@@ -0,0 +1,31 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.MFA.BackupCodes do
+  @moduledoc """
+  This module contains functions for generating backup codes.
+  """
+  alias Pleroma.Config
+
+  @config_ns [:instance, :multi_factor_authentication, :backup_codes]
+
+  @doc """
+  Generates backup codes.
+  """
+  @spec generate(Keyword.t()) :: list(String.t())
+  def generate(opts \\ []) do
+    number_of_codes = Keyword.get(opts, :number_of_codes, default_backup_codes_number())
+    code_length = Keyword.get(opts, :length, default_backup_codes_code_length())
+
+    Enum.map(1..number_of_codes, fn _ ->
+      :crypto.strong_rand_bytes(div(code_length, 2))
+      |> Base.encode16(case: :lower)
+    end)
+  end
+
+  defp default_backup_codes_number, do: Config.get(@config_ns ++ [:number], 5)
+
+  defp default_backup_codes_code_length,
+    do: Config.get(@config_ns ++ [:length], 16)
+end

lib/pleroma/mfa/changeset.ex

@@ -0,0 +1,64 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.MFA.Changeset do
+  alias Pleroma.MFA
+  alias Pleroma.MFA.Settings
+  alias Pleroma.User
+
+  def disable(%Ecto.Changeset{} = changeset, force \\ false) do
+    settings =
+      changeset
+      |> Ecto.Changeset.apply_changes()
+      |> MFA.fetch_settings()
+
+    if force || not MFA.enabled?(settings) do
+      put_change(changeset, %Settings{settings | enabled: false})
+    else
+      changeset
+    end
+  end
+
+  def disable_totp(%User{multi_factor_authentication_settings: settings} = user) do
+    user
+    |> put_change(%Settings{settings | totp: %Settings.TOTP{}})
+  end
+
+  def confirm_totp(%User{multi_factor_authentication_settings: settings} = user) do
+    totp_settings = %Settings.TOTP{settings.totp | confirmed: true}
+
+    user
+    |> put_change(%Settings{settings | totp: totp_settings, enabled: true})
+  end
+
+  def setup_totp(%User{} = user, attrs) do
+    mfa_settings = MFA.fetch_settings(user)
+
+    totp_settings =
+      %Settings.TOTP{}
+      |> Ecto.Changeset.cast(attrs, [:secret, :delivery_type])
+
+    user
+    |> put_change(%Settings{mfa_settings | totp: Ecto.Changeset.apply_changes(totp_settings)})
+  end
+
+  def cast_backup_codes(%User{} = user, codes) do
+    user
+    |> put_change(%Settings{
+      user.multi_factor_authentication_settings
+      | backup_codes: codes
+    })
+  end
+
+  defp put_change(%User{} = user, settings) do
+    user
+    |> Ecto.Changeset.change()
+    |> put_change(settings)
+  end
+
+  defp put_change(%Ecto.Changeset{} = changeset, settings) do
+    changeset
+    |> Ecto.Changeset.put_change(:multi_factor_authentication_settings, settings)
+  end
+end

lib/pleroma/mfa/settings.ex

@@ -0,0 +1,24 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.MFA.Settings do
+  use Ecto.Schema
+
+  @primary_key false
+
+  @mfa_methods [:totp]
+  embedded_schema do
+    field(:enabled, :boolean, default: false)
+    field(:backup_codes, {:array, :string}, default: [])
+
+    embeds_one :totp, TOTP, on_replace: :delete, primary_key: false do
+      field(:secret, :string)
+      # app | sms
+      field(:delivery_type, :string, default: "app")
+      field(:confirmed, :boolean, default: false)
+    end
+  end
+
+  def mfa_methods, do: @mfa_methods
+end

lib/pleroma/mfa/token.ex

@@ -0,0 +1,106 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.MFA.Token do
+  use Ecto.Schema
+  import Ecto.Query
+  import Ecto.Changeset
+
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Web.OAuth.Authorization
+  alias Pleroma.Web.OAuth.Token, as: OAuthToken
+
+  @expires 300
+
+  schema "mfa_tokens" do
+    field(:token, :string)
+    field(:valid_until, :naive_datetime_usec)
+
+    belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
+    belongs_to(:authorization, Authorization)
+
+    timestamps()
+  end
+
+  def get_by_token(token) do
+    from(
+      t in __MODULE__,
+      where: t.token == ^token,
+      preload: [:user, :authorization]
+    )
+    |> Repo.find_resource()
+  end
+
+  def validate(token) do
+    with {:fetch_token, {:ok, token}} <- {:fetch_token, get_by_token(token)},
+         {:expired, false} <- {:expired, is_expired?(token)} do
+      {:ok, token}
+    else
+      {:expired, _} -> {:error, :expired_token}
+      {:fetch_token, _} -> {:error, :not_found}
+      error -> {:error, error}
+    end
+  end
+
+  def create_token(%User{} = user) do
+    %__MODULE__{}
+    |> change
+    |> assign_user(user)
+    |> put_token
+    |> put_valid_until
+    |> Repo.insert()
+  end
+
+  def create_token(user, authorization) do
+    %__MODULE__{}
+    |> change
+    |> assign_user(user)
+    |> assign_authorization(authorization)
+    |> put_token
+    |> put_valid_until
+    |> Repo.insert()
+  end
+
+  defp assign_user(changeset, user) do
+    changeset
+    |> put_assoc(:user, user)
+    |> validate_required([:user])
+  end
+
+  defp assign_authorization(changeset, authorization) do
+    changeset
+    |> put_assoc(:authorization, authorization)
+    |> validate_required([:authorization])
+  end
+
+  defp put_token(changeset) do
+    changeset
+    |> change(%{token: OAuthToken.Utils.generate_token()})
+    |> validate_required([:token])
+    |> unique_constraint(:token)
+  end
+
+  defp put_valid_until(changeset) do
+    expires_in = NaiveDateTime.add(NaiveDateTime.utc_now(), @expires)
+
+    changeset
+    |> change(%{valid_until: expires_in})
+    |> validate_required([:valid_until])
+  end
+
+  def is_expired?(%__MODULE__{valid_until: valid_until}) do
+    NaiveDateTime.diff(NaiveDateTime.utc_now(), valid_until) > 0
+  end
+
+  def is_expired?(_), do: false
+
+  def delete_expired_tokens do
+    from(
+      q in __MODULE__,
+      where: fragment("?", q.valid_until) < ^Timex.now()
+    )
+    |> Repo.delete_all()
+  end
+end

lib/pleroma/mfa/totp.ex

@@ -0,0 +1,86 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.MFA.TOTP do
+  @moduledoc """
+  This module represents functions to create secrets for
+  TOTP Application as well as validate them with a time based token.
+  """
+  alias Pleroma.Config
+
+  @config_ns [:instance, :multi_factor_authentication, :totp]
+
+  @doc """
+  https://github.com/google/google-authenticator/wiki/Key-Uri-Format
+  """
+  def provisioning_uri(secret, label, opts \\ []) do
+    query =
+      %{
+        secret: secret,
+        issuer: Keyword.get(opts, :issuer, default_issuer()),
+        digits: Keyword.get(opts, :digits, default_digits()),
+        period: Keyword.get(opts, :period, default_period())
+      }
+      |> Enum.filter(fn {_, v} -> not is_nil(v) end)
+      |> Enum.into(%{})
+      |> URI.encode_query()
+
+    %URI{scheme: "otpauth", host: "totp", path: "/" <> label, query: query}
+    |> URI.to_string()
+  end
+
+  defp default_period, do: Config.get(@config_ns ++ [:period])
+  defp default_digits, do: Config.get(@config_ns ++ [:digits])
+
+  defp default_issuer,
+    do: Config.get(@config_ns ++ [:issuer], Config.get([:instance, :name]))
+
+  @doc "Creates a random Base 32 encoded string"
+  def generate_secret do
+    Base.encode32(:crypto.strong_rand_bytes(10))
+  end
+
+  @doc "Generates a valid token based on a secret"
+  def generate_token(secret) do
+    :pot.totp(secret)
+  end
+
+  @doc """
+  Validates a given token based on a secret.
+
+  optional parameters:
+  `token_length` default `6`
+  `interval_length` default `30`
+  `window` default 0
+
+  Returns {:ok, :pass} if the token is valid and
+  {:error, :invalid_token} if it is not.
+  """
+  @spec validate_token(String.t(), String.t()) ::
+          {:ok, :pass} | {:error, :invalid_token | :invalid_secret_and_token}
+  def validate_token(secret, token)
+      when is_binary(secret) and is_binary(token) do
+    opts = [
+      token_length: default_digits(),
+      interval_length: default_period()
+    ]
+
+    validate_token(secret, token, opts)
+  end
+
+  def validate_token(_, _), do: {:error, :invalid_secret_and_token}
+
+  @doc "See `validate_token/2`"
+  @spec validate_token(String.t(), String.t(), Keyword.t()) ::
+          {:ok, :pass} | {:error, :invalid_token | :invalid_secret_and_token}
+  def validate_token(secret, token, options)
+      when is_binary(secret) and is_binary(token) do
+    case :pot.valid_totp(token, secret, options) do
+      true -> {:ok, :pass}
+      false -> {:error, :invalid_token}
+    end
+  end
+
+  def validate_token(_, _, _), do: {:error, :invalid_secret_and_token}
+end

lib/pleroma/moderation_log.ex

@@ -605,6 +605,17 @@ def get_log_entry_message(%ModerationLog{
     }"
   end
 
+  @spec get_log_entry_message(ModerationLog) :: String.t()
+  def get_log_entry_message(%ModerationLog{
+        data: %{
+          "actor" => %{"nickname" => actor_nickname},
+          "action" => "updated_users",
+          "subject" => subjects
+        }
+      }) do
+    "@#{actor_nickname} updated users: #{users_to_nicknames_string(subjects)}"
+  end
+
   defp nicknames_to_string(nicknames) do
     nicknames
     |> Enum.map(&"@#{&1}")

lib/pleroma/notification.ex

@@ -5,11 +5,15 @@
 defmodule Pleroma.Notification do
   use Ecto.Schema
 
+  alias Ecto.Multi
   alias Pleroma.Activity
+  alias Pleroma.FollowingRelationship
+  alias Pleroma.Marker
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Pagination
   alias Pleroma.Repo
+  alias Pleroma.ThreadMute
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.Push
@@ -17,6 +21,7 @@ defmodule Pleroma.Notification do
 
   import Ecto.Query
   import Ecto.Changeset
+
   require Logger
 
   @type t :: %__MODULE__{}
@@ -31,17 +36,36 @@ defmodule Pleroma.Notification do
     timestamps()
   end
 
+  @spec unread_notifications_count(User.t()) :: integer()
+  def unread_notifications_count(%User{id: user_id}) do
+    from(q in __MODULE__,
+      where: q.user_id == ^user_id and q.seen == false
+    )
+    |> Repo.aggregate(:count, :id)
+  end
+
   def changeset(%Notification{} = notification, attrs) do
     notification
     |> cast(attrs, [:seen])
   end
 
+  @spec last_read_query(User.t()) :: Ecto.Queryable.t()
+  def last_read_query(user) do
+    from(q in Pleroma.Notification,
+      where: q.user_id == ^user.id,
+      where: q.seen == true,
+      select: type(q.id, :string),
+      limit: 1,
+      order_by: [desc: :id]
+    )
+  end
+
   defp for_user_query_ap_id_opts(user, opts) do
-    ap_id_relations =
+    ap_id_relationships =
       [:block] ++
         if opts[@include_muted_option], do: [], else: [:notification_mute]
 
-    preloaded_ap_ids = User.outgoing_relations_ap_ids(user, ap_id_relations)
+    preloaded_ap_ids = User.outgoing_relationships_ap_ids(user, ap_id_relationships)
 
     exclude_blocked_opts = Map.merge(%{blocked_users_ap_ids: preloaded_ap_ids[:block]}, opts)
 
@@ -79,15 +103,13 @@ def for_user_query(user, opts \\ %{}) do
     |> exclude_visibility(opts)
   end
 
+  # Excludes blocked users and non-followed domain-blocked users
   defp exclude_blocked(query, user, opts) do
     blocked_ap_ids = opts[:blocked_users_ap_ids] || User.blocked_users_ap_ids(user)
 
     query
     |> where([n, a], a.actor not in ^blocked_ap_ids)
-    |> where(
+    |> FollowingRelationship.keep_following_or_not_domain_blocked(user)
-      [n, a],
-      fragment("substring(? from '.*://([^/]*)')", a.actor) not in ^user.domain_blocks
-    )
   end
 
   defp exclude_notification_muted(query, _, %{@include_muted_option => true}) do
@@ -100,7 +122,7 @@ defp exclude_notification_muted(query, user, opts) do
 
     query
     |> where([n, a], a.actor not in ^notification_muted_ap_ids)
-    |> join(:left, [n, a], tm in Pleroma.ThreadMute,
+    |> join(:left, [n, a], tm in ThreadMute,
       on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data)
     )
     |> where([n, a, o, tm], is_nil(tm.user_id))
@@ -184,25 +206,23 @@ def for_user_since(user, date) do
     |> Repo.all()
   end
 
-  def set_read_up_to(%{id: user_id} = _user, id) do
+  def set_read_up_to(%{id: user_id} = user, id) do
     query =
       from(
         n in Notification,
         where: n.user_id == ^user_id,
         where: n.id <= ^id,
         where: n.seen == false,
-        update: [
-          set: [
-            seen: true,
-            updated_at: ^NaiveDateTime.utc_now()
-          ]
-        ],
         # Ideally we would preload object and activities here
         # but Ecto does not support preloads in update_all
         select: n.id
       )
 
-    {_, notification_ids} = Repo.update_all(query, [])
+    {:ok, %{ids: {_, notification_ids}}} =
+      Multi.new()
+      |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()])
+      |> Marker.multi_set_last_read_id(user, "notifications")
+      |> Repo.transaction()
 
     Notification
     |> where([n], n.id in ^notification_ids)
@@ -219,11 +239,18 @@ def set_read_up_to(%{id: user_id} = _user, id) do
     |> Repo.all()
   end
 
+  @spec read_one(User.t(), String.t()) ::
+          {:ok, Notification.t()} | {:error, Ecto.Changeset.t()} | nil
   def read_one(%User{} = user, notification_id) do
     with {:ok, %Notification{} = notification} <- get(user, notification_id) do
-      notification
+      Multi.new()
-      |> changeset(%{seen: true})
+      |> Multi.update(:update, changeset(notification, %{seen: true}))
-      |> Repo.update()
+      |> Marker.multi_set_last_read_id(user, "notifications")
+      |> Repo.transaction()
+      |> case do
+        {:ok, %{update: notification}} -> {:ok, notification}
+        {:error, :update, changeset, _} -> {:error, changeset}
+      end
     end
   end
 
@@ -260,6 +287,16 @@ def destroy_multiple(%{id: user_id} = _user, ids) do
     |> Repo.delete_all()
   end
 
+  def dismiss(%Pleroma.Activity{} = activity) do
+    Notification
+    |> where([n], n.activity_id == ^activity.id)
+    |> Repo.delete_all()
+    |> case do
+      {_, notifications} -> {:ok, notifications}
+      _ -> {:error, "Cannot dismiss notification"}
+    end
+  end
+
   def dismiss(%{id: user_id} = _user, id) do
     notification = Repo.get(Notification, id)
 
@@ -275,58 +312,159 @@ def dismiss(%{id: user_id} = _user, id) do
   def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity) do
     object = Object.normalize(activity)
 
-    unless object && object.data["type"] == "Answer" do
+    if object && object.data["type"] == "Answer" do
-      users = get_notified_from_activity(activity)
-      notifications = Enum.map(users, fn user -> create_notification(activity, user) end)
-      {:ok, notifications}
-    else
       {:ok, []}
+    else
+      do_create_notifications(activity)
     end
   end
 
   def create_notifications(%Activity{data: %{"type" => type}} = activity)
-      when type in ["Like", "Announce", "Follow", "Move", "EmojiReact"] do
+      when type in ["Follow", "Like", "Announce", "Move", "EmojiReact"] do
-    notifications =
+    do_create_notifications(activity)
-      activity
-      |> get_notified_from_activity()
-      |> Enum.map(&create_notification(activity, &1))
-
-    {:ok, notifications}
   end
 
   def create_notifications(_), do: {:ok, []}
 
+  defp do_create_notifications(%Activity{} = activity) do
+    {enabled_receivers, disabled_receivers} = get_notified_from_activity(activity)
+    potential_receivers = enabled_receivers ++ disabled_receivers
+
+    notifications =
+      Enum.map(potential_receivers, fn user ->
+        do_send = user in enabled_receivers
+        create_notification(activity, user, do_send)
+      end)
+
+    {:ok, notifications}
+  end
+
   # TODO move to sql, too.
-  def create_notification(%Activity{} = activity, %User{} = user) do
+  def create_notification(%Activity{} = activity, %User{} = user, do_send \\ true) do
     unless skip?(activity, user) do
-      notification = %Notification{user_id: user.id, activity: activity}
+      {:ok, %{notification: notification}} =
-      {:ok, notification} = Repo.insert(notification)
+        Multi.new()
+        |> Multi.insert(:notification, %Notification{user_id: user.id, activity: activity})
+        |> Marker.multi_set_last_read_id(user, "notifications")
+        |> Repo.transaction()
 
-      ["user", "user:notification"]
+      if do_send do
-      |> Streamer.stream(notification)
+        Streamer.stream(["user", "user:notification"], notification)
+        Push.send(notification)
+      end
 
-      Push.send(notification)
       notification
     end
   end
 
+  @doc """
+  Returns a tuple with 2 elements:
+    {notification-enabled receivers, currently disabled receivers (blocking / [thread] muting)}
+
+  NOTE: might be called for FAKE Activities, see ActivityPub.Utils.get_notified_from_object/1
+  """
+  @spec get_notified_from_activity(Activity.t(), boolean()) :: {list(User.t()), list(User.t())}
   def get_notified_from_activity(activity, local_only \\ true)
 
   def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only)
       when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact"] do
+    potential_receiver_ap_ids = get_potential_receiver_ap_ids(activity)
+
+    potential_receivers = User.get_users_from_set(potential_receiver_ap_ids, local_only)
+
+    notification_enabled_ap_ids =
+      potential_receiver_ap_ids
+      |> exclude_domain_blocker_ap_ids(activity, potential_receivers)
+      |> exclude_relationship_restricted_ap_ids(activity)
+      |> exclude_thread_muter_ap_ids(activity)
+
+    notification_enabled_users =
+      Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end)
+
+    {notification_enabled_users, potential_receivers -- notification_enabled_users}
+  end
+
+  def get_notified_from_activity(_, _local_only), do: {[], []}
+
+  # For some activities, only notify the author of the object
+  def get_potential_receiver_ap_ids(%{data: %{"type" => type, "object" => object_id}})
+      when type in ~w{Like Announce EmojiReact} do
+    case Object.get_cached_by_ap_id(object_id) do
+      %Object{data: %{"actor" => actor}} ->
+        [actor]
+
+      _ ->
+        []
+    end
+  end
+
+  def get_potential_receiver_ap_ids(activity) do
     []
     |> Utils.maybe_notify_to_recipients(activity)
     |> Utils.maybe_notify_mentioned_recipients(activity)
     |> Utils.maybe_notify_subscribers(activity)
     |> Utils.maybe_notify_followers(activity)
     |> Enum.uniq()
-    |> User.get_users_from_set(local_only)
   end
 
-  def get_notified_from_activity(_, _local_only), do: []
+  @doc "Filters out AP IDs domain-blocking and not following the activity's actor"
+  def exclude_domain_blocker_ap_ids(ap_ids, activity, preloaded_users \\ [])
+
+  def exclude_domain_blocker_ap_ids([], _activity, _preloaded_users), do: []
+
+  def exclude_domain_blocker_ap_ids(ap_ids, %Activity{} = activity, preloaded_users) do
+    activity_actor_domain = activity.actor && URI.parse(activity.actor).host
+
+    users =
+      ap_ids
+      |> Enum.map(fn ap_id ->
+        Enum.find(preloaded_users, &(&1.ap_id == ap_id)) ||
+          User.get_cached_by_ap_id(ap_id)
+      end)
+      |> Enum.filter(& &1)
+
+    domain_blocker_ap_ids = for u <- users, activity_actor_domain in u.domain_blocks, do: u.ap_id
+
+    domain_blocker_follower_ap_ids =
+      if Enum.any?(domain_blocker_ap_ids) do
+        activity
+        |> Activity.user_actor()
+        |> FollowingRelationship.followers_ap_ids(domain_blocker_ap_ids)
+      else
+        []
+      end
+
+    ap_ids
+    |> Kernel.--(domain_blocker_ap_ids)
+    |> Kernel.++(domain_blocker_follower_ap_ids)
+  end
+
+  @doc "Filters out AP IDs of users basing on their relationships with activity actor user"
+  def exclude_relationship_restricted_ap_ids([], _activity), do: []
+
+  def exclude_relationship_restricted_ap_ids(ap_ids, %Activity{} = activity) do
+    relationship_restricted_ap_ids =
+      activity
+      |> Activity.user_actor()
+      |> User.incoming_relationships_ungrouped_ap_ids([
+        :block,
+        :notification_mute
+      ])
+
+    Enum.uniq(ap_ids) -- relationship_restricted_ap_ids
+  end
+
+  @doc "Filters out AP IDs of users who mute activity thread"
+  def exclude_thread_muter_ap_ids([], _activity), do: []
+
+  def exclude_thread_muter_ap_ids(ap_ids, %Activity{} = activity) do
+    thread_muter_ap_ids = ThreadMute.muter_ap_ids(activity.data["context"])
+
+    Enum.uniq(ap_ids) -- thread_muter_ap_ids
+  end
 
   @spec skip?(Activity.t(), User.t()) :: boolean()
-  def skip?(activity, user) do
+  def skip?(%Activity{} = activity, %User{} = user) do
     [
       :self,
       :followers,
@@ -335,18 +473,20 @@ def skip?(activity, user) do
       :non_follows,
       :recently_followed
     ]
-    |> Enum.any?(&skip?(&1, activity, user))
+    |> Enum.find(&skip?(&1, activity, user))
   end
 
+  def skip?(_, _), do: false
+
   @spec skip?(atom(), Activity.t(), User.t()) :: boolean()
-  def skip?(:self, activity, user) do
+  def skip?(:self, %Activity{} = activity, %User{} = user) do
     activity.data["actor"] == user.ap_id
   end
 
   def skip?(
         :followers,
-        activity,
+        %Activity{} = activity,
-        %{notification_settings: %{followers: false}} = user
+        %User{notification_settings: %{followers: false}} = user
       ) do
     actor = activity.data["actor"]
     follower = User.get_cached_by_ap_id(actor)
@@ -355,15 +495,19 @@ def skip?(
 
   def skip?(
         :non_followers,
-        activity,
+        %Activity{} = activity,
-        %{notification_settings: %{non_followers: false}} = user
+        %User{notification_settings: %{non_followers: false}} = user
       ) do
     actor = activity.data["actor"]
     follower = User.get_cached_by_ap_id(actor)
     !User.following?(follower, user)
   end
 
-  def skip?(:follows, activity, %{notification_settings: %{follows: false}} = user) do
+  def skip?(
+        :follows,
+        %Activity{} = activity,
+        %User{notification_settings: %{follows: false}} = user
+      ) do
     actor = activity.data["actor"]
     followed = User.get_cached_by_ap_id(actor)
     User.following?(user, followed)
@@ -371,15 +515,16 @@ def skip?(:follows, activity, %{notification_settings: %{follows: false}} = user
 
   def skip?(
         :non_follows,
-        activity,
+        %Activity{} = activity,
-        %{notification_settings: %{non_follows: false}} = user
+        %User{notification_settings: %{non_follows: false}} = user
       ) do
     actor = activity.data["actor"]
     followed = User.get_cached_by_ap_id(actor)
     !User.following?(user, followed)
   end
 
-  def skip?(:recently_followed, %{data: %{"type" => "Follow"}} = activity, user) do
+  # To do: consider defining recency in hours and checking FollowingRelationship with a single SQL
+  def skip?(:recently_followed, %Activity{data: %{"type" => "Follow"}} = activity, %User{} = user) do
     actor = activity.data["actor"]
 
     Notification.for_user(user)

lib/pleroma/object.ex

@@ -261,7 +261,7 @@ def decrease_replies_count(ap_id) do
     end
   end
 
-  def increase_vote_count(ap_id, name) do
+  def increase_vote_count(ap_id, name, actor) do
     with %Object{} = object <- Object.normalize(ap_id),
          "Question" <- object.data["type"] do
       multiple = Map.has_key?(object.data, "anyOf")
@@ -276,12 +276,15 @@ def increase_vote_count(ap_id, name) do
             option
         end)
 
+      voters = [actor | object.data["voters"] || []] |> Enum.uniq()
+
       data =
         if multiple do
           Map.put(object.data, "anyOf", options)
         else
           Map.put(object.data, "oneOf", options)
         end
+        |> Map.put("voters", voters)
 
       object
       |> Object.change(%{data: data})

lib/pleroma/object/containment.ex

@@ -32,6 +32,18 @@ def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor)
     get_actor(%{"actor" => actor})
   end
 
+  def get_object(%{"object" => id}) when is_binary(id) do
+    id
+  end
+
+  def get_object(%{"object" => %{"id" => id}}) when is_binary(id) do
+    id
+  end
+
+  def get_object(_) do
+    nil
+  end
+
   # TODO: We explicitly allow 'tag' URIs through, due to references to legacy OStatus
   # objects being present in the test suite environment.  Once these objects are
   # removed, please also remove this.

lib/pleroma/object/fetcher.ex

@@ -141,7 +141,7 @@ defp make_signature(id, date) do
         date: date
       })
 
-    [{:Signature, signature}]
+    [{"signature", signature}]
   end
 
   defp sign_fetch(headers, id, date) do
@@ -154,7 +154,7 @@ defp sign_fetch(headers, id, date) do
 
   defp maybe_date_fetch(headers, date) do
     if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
-      headers ++ [{:Date, date}]
+      headers ++ [{"date", date}]
     else
       headers
     end
@@ -166,7 +166,7 @@ def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
     date = Pleroma.Signature.signed_date()
 
     headers =
-      [{:Accept, "application/activity+json"}]
+      [{"accept", "application/activity+json"}]
       |> maybe_date_fetch(date)
       |> sign_fetch(id, date)
 

lib/pleroma/otp_version.ex

@@ -0,0 +1,28 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.OTPVersion do
+  @spec version() :: String.t() | nil
+  def version do
+    # OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version
+    [
+      Path.join(:code.root_dir(), "OTP_VERSION"),
+      Path.join([:code.root_dir(), "releases", :erlang.system_info(:otp_release), "OTP_VERSION"])
+    ]
+    |> get_version_from_files()
+  end
+
+  @spec get_version_from_files([Path.t()]) :: String.t() | nil
+  def get_version_from_files([]), do: nil
+
+  def get_version_from_files([path | paths]) do
+    if File.exists?(path) do
+      path
+      |> File.read!()
+      |> String.replace(~r/\r|\n|\s/, "")
+    else
+      get_version_from_files(paths)
+    end
+  end
+end

lib/pleroma/plugs/authentication_plug.ex

@@ -3,9 +3,11 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.AuthenticationPlug do
-  alias Comeonin.Pbkdf2
+  alias Pleroma.Plugs.OAuthScopesPlug
-  import Plug.Conn
   alias Pleroma.User
+
+  import Plug.Conn
+
   require Logger
 
   def init(options), do: options
@@ -14,8 +16,13 @@ def checkpw(password, "$6" <> _ = password_hash) do
     :crypt.crypt(password, password_hash) == password_hash
   end
 
+  def checkpw(password, "$2" <> _ = password_hash) do
+    # Handle bcrypt passwords for Mastodon migration
+    Bcrypt.verify_pass(password, password_hash)
+  end
+
   def checkpw(password, "$pbkdf2" <> _ = password_hash) do
-    Pbkdf2.checkpw(password, password_hash)
+    Pbkdf2.verify_pass(password, password_hash)
   end
 
   def checkpw(_password, _password_hash) do
@@ -34,16 +41,17 @@ def call(
         } = conn,
         _
       ) do
-    if Pbkdf2.checkpw(password, password_hash) do
+    if checkpw(password, password_hash) do
       conn
       |> assign(:user, auth_user)
+      |> OAuthScopesPlug.skip_plug()
     else
       conn
     end
   end
 
   def call(%{assigns: %{auth_credentials: %{password: _}}} = conn, _) do
-    Pbkdf2.dummy_checkpw()
+    Pbkdf2.no_user_verify()
     conn
   end
 

lib/pleroma/plugs/ensure_authenticated_plug.ex

@@ -5,32 +5,35 @@
 defmodule Pleroma.Plugs.EnsureAuthenticatedPlug do
   import Plug.Conn
   import Pleroma.Web.TranslationHelpers
+
   alias Pleroma.User
 
+  use Pleroma.Web, :plug
+
   def init(options) do
     options
   end
 
-  def call(%{assigns: %{user: %User{}}} = conn, _) do
+  @impl true
+  def perform(
+        %{
+          assigns: %{
+            auth_credentials: %{password: _},
+            user: %User{multi_factor_authentication_settings: %{enabled: true}}
+          }
+        } = conn,
+        _
+      ) do
+    conn
+    |> render_error(:forbidden, "Two-factor authentication enabled, you must use a access token.")
+    |> halt()
+  end
+
+  def perform(%{assigns: %{user: %User{}}} = conn, _) do
     conn
   end
 
-  def call(conn, options) do
+  def perform(conn, _) do
-    perform =
-      cond do
-        options[:if_func] -> options[:if_func].()
-        options[:unless_func] -> !options[:unless_func].()
-        true -> true
-      end
-
-    if perform do
-      fail(conn)
-    else
-      conn
-    end
-  end
-
-  def fail(conn) do
     conn
     |> render_error(:forbidden, "Invalid credentials.")
     |> halt()

lib/pleroma/plugs/ensure_public_or_authenticated_plug.ex

@@ -5,14 +5,18 @@
 defmodule Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug do
   import Pleroma.Web.TranslationHelpers
   import Plug.Conn
+
   alias Pleroma.Config
   alias Pleroma.User
 
+  use Pleroma.Web, :plug
+
   def init(options) do
     options
   end
 
-  def call(conn, _) do
+  @impl true
+  def perform(conn, _) do
     public? = Config.get!([:instance, :public])
 
     case {public?, conn} do

lib/pleroma/plugs/expect_authenticated_check_plug.ex

@@ -0,0 +1,20 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Plugs.ExpectAuthenticatedCheckPlug do
+  @moduledoc """
+  Marks `Pleroma.Plugs.EnsureAuthenticatedPlug` as expected to be executed later in plug chain.
+
+  No-op plug which affects `Pleroma.Web` operation (is checked with `PlugHelper.plug_called?/2`).
+  """
+
+  use Pleroma.Web, :plug
+
+  def init(options), do: options
+
+  @impl true
+  def perform(conn, _) do
+    conn
+  end
+end

lib/pleroma/plugs/expect_public_or_authenticated_check_plug.ex

@@ -0,0 +1,21 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Plugs.ExpectPublicOrAuthenticatedCheckPlug do
+  @moduledoc """
+  Marks `Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug` as expected to be executed later in plug
+  chain.
+
+  No-op plug which affects `Pleroma.Web` operation (is checked with `PlugHelper.plug_called?/2`).
+  """
+
+  use Pleroma.Web, :plug
+
+  def init(options), do: options
+
+  @impl true
+  def perform(conn, _) do
+    conn
+  end
+end

lib/pleroma/plugs/federating_plug.ex

@@ -19,6 +19,9 @@ def call(conn, _opts) do
 
   def federating?, do: Pleroma.Config.get([:instance, :federating])
 
+  # Definition for the use in :if_func / :unless_func plug options
+  def federating?(_conn), do: federating?()
+
   defp fail(conn) do
     conn
     |> put_status(404)

lib/pleroma/plugs/http_security_plug.ex

@@ -75,7 +75,7 @@ defp csp_string do
       "default-src 'none'",
       "base-uri 'self'",
       "frame-ancestors 'none'",
-      "img-src 'self' data: https:",
+      "img-src 'self' data: blob: https:",
       "media-src 'self' https:",
       "style-src 'self' 'unsafe-inline'",
       "font-src 'self'",

lib/pleroma/plugs/instance_static.ex

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.InstanceStatic do
+  require Pleroma.Constants
+
   @moduledoc """
   This is a shim to call `Plug.Static` but with runtime `from` configuration.
 
@@ -21,9 +23,6 @@ def file_path(path) do
     end
   end
 
-  @only ~w(index.html robots.txt static emoji packs sounds images instance favicon.png sw.js
-  sw-pleroma.js)
-
   def init(opts) do
     opts
     |> Keyword.put(:from, "__unconfigured_instance_static_plug")
@@ -31,7 +30,7 @@ def init(opts) do
     |> Plug.Static.init()
   end
 
-  for only <- @only do
+  for only <- Pleroma.Constants.static_only_files() do
     at = Plug.Router.Utils.split("/")
 
     def call(%{request_path: "/" <> unquote(only) <> _} = conn, opts) do

lib/pleroma/plugs/legacy_authentication_plug.ex

@@ -4,6 +4,8 @@
 
 defmodule Pleroma.Plugs.LegacyAuthenticationPlug do
   import Plug.Conn
+
+  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.User
 
   def init(options) do
@@ -27,6 +29,7 @@ def call(
       conn
       |> assign(:auth_user, user)
       |> assign(:user, user)
+      |> OAuthScopesPlug.skip_plug()
     else
       _ ->
         conn

lib/pleroma/plugs/mapped_signature_to_identity_plug.ex

@@ -13,8 +13,9 @@ defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlug do
   def init(options), do: options
 
   defp key_id_from_conn(conn) do
-    with %{"keyId" => key_id} <- HTTPSignatures.signature_for_conn(conn) do
+    with %{"keyId" => key_id} <- HTTPSignatures.signature_for_conn(conn),
-      Signature.key_id_to_actor_id(key_id)
+         {:ok, ap_id} <- Signature.key_id_to_actor_id(key_id) do
+      ap_id
     else
       _ ->
         nil
@@ -42,13 +43,13 @@ def call(%{assigns: %{valid_signature: true}, params: %{"actor" => actor}} = con
     else
       {:user_match, false} ->
         Logger.debug("Failed to map identity from signature (payload actor mismatch)")
-        Logger.debug("key_id=#{key_id_from_conn(conn)}, actor=#{actor}")
+        Logger.debug("key_id=#{inspect(key_id_from_conn(conn))}, actor=#{inspect(actor)}")
         assign(conn, :valid_signature, false)
 
       # remove me once testsuite uses mapped capabilities instead of what we do now
       {:user, nil} ->
         Logger.debug("Failed to map identity from signature (lookup failure)")
-        Logger.debug("key_id=#{key_id_from_conn(conn)}, actor=#{actor}")
+        Logger.debug("key_id=#{inspect(key_id_from_conn(conn))}, actor=#{actor}")
         conn
     end
   end
@@ -60,7 +61,7 @@ def call(%{assigns: %{valid_signature: true}} = conn, _opts) do
     else
       _ ->
         Logger.debug("Failed to map identity from signature (no payload actor mismatch)")
-        Logger.debug("key_id=#{key_id_from_conn(conn)}")
+        Logger.debug("key_id=#{inspect(key_id_from_conn(conn))}")
         assign(conn, :valid_signature, false)
     end
   end

lib/pleroma/plugs/oauth_scopes_plug.ex

@@ -7,13 +7,13 @@ defmodule Pleroma.Plugs.OAuthScopesPlug do
   import Pleroma.Web.Gettext
 
   alias Pleroma.Config
-  alias Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug
 
-  @behaviour Plug
+  use Pleroma.Web, :plug
 
   def init(%{scopes: _} = options), do: options
 
-  def call(%Plug.Conn{assigns: assigns} = conn, %{scopes: scopes} = options) do
+  @impl true
+  def perform(%Plug.Conn{assigns: assigns} = conn, %{scopes: scopes} = options) do
     op = options[:op] || :|
     token = assigns[:token]
 
@@ -28,10 +28,7 @@ def call(%Plug.Conn{assigns: assigns} = conn, %{scopes: scopes} = options) do
         conn
 
       options[:fallback] == :proceed_unauthenticated ->
-        conn
+        drop_auth_info(conn)
-        |> assign(:user, nil)
-        |> assign(:token, nil)
-        |> maybe_perform_instance_privacy_check(options)
 
       true ->
         missing_scopes = scopes -- matched_scopes
@@ -47,6 +44,15 @@ def call(%Plug.Conn{assigns: assigns} = conn, %{scopes: scopes} = options) do
     end
   end
 
+  @doc "Drops authentication info from connection"
+  def drop_auth_info(conn) do
+    # To simplify debugging, setting a private variable on `conn` if auth info is dropped
+    conn
+    |> put_private(:authentication_ignored, true)
+    |> assign(:user, nil)
+    |> assign(:token, nil)
+  end
+
   @doc "Filters descendants of supported scopes"
   def filter_descendants(scopes, supported_scopes) do
     Enum.filter(
@@ -68,12 +74,4 @@ def transform_scopes(scopes, options) do
       scopes
     end
   end
-
-  defp maybe_perform_instance_privacy_check(%Plug.Conn{} = conn, options) do
-    if options[:skip_instance_privacy_check] do
-      conn
-    else
-      EnsurePublicOrAuthenticatedPlug.call(conn, [])
-    end
-  end
 end

lib/pleroma/plugs/plug_helper.ex

@@ -0,0 +1,40 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Plugs.PlugHelper do
+  @moduledoc "Pleroma Plug helper"
+
+  @called_plugs_list_id :called_plugs
+  def called_plugs_list_id, do: @called_plugs_list_id
+
+  @skipped_plugs_list_id :skipped_plugs
+  def skipped_plugs_list_id, do: @skipped_plugs_list_id
+
+  @doc "Returns `true` if specified plug was called."
+  def plug_called?(conn, plug_module) do
+    contained_in_private_list?(conn, @called_plugs_list_id, plug_module)
+  end
+
+  @doc "Returns `true` if specified plug was explicitly marked as skipped."
+  def plug_skipped?(conn, plug_module) do
+    contained_in_private_list?(conn, @skipped_plugs_list_id, plug_module)
+  end
+
+  @doc "Returns `true` if specified plug was either called or explicitly marked as skipped."
+  def plug_called_or_skipped?(conn, plug_module) do
+    plug_called?(conn, plug_module) || plug_skipped?(conn, plug_module)
+  end
+
+  # Appends plug to known list (skipped, called). Intended to be used from within plug code only.
+  def append_to_private_list(conn, list_id, value) do
+    list = conn.private[list_id] || []
+    modified_list = Enum.uniq(list ++ [value])
+    Plug.Conn.put_private(conn, list_id, modified_list)
+  end
+
+  defp contained_in_private_list?(conn, private_variable, value) do
+    list = conn.private[private_variable] || []
+    value in list
+  end
+end

lib/pleroma/plugs/rate_limiter/rate_limiter.ex

@@ -110,20 +110,9 @@ defp handle(conn, action_settings) do
   end
 
   def disabled?(conn) do
-    localhost_or_socket =
+    if Map.has_key?(conn.assigns, :remote_ip_found),
-      case Config.get([Pleroma.Web.Endpoint, :http, :ip]) do
+      do: !conn.assigns.remote_ip_found,
-        {127, 0, 0, 1} -> true
+      else: false
-        {0, 0, 0, 0, 0, 0, 0, 1} -> true
-        {:local, _} -> true
-        _ -> false
-      end
-
-    remote_ip_not_found =
-      if Map.has_key?(conn.assigns, :remote_ip_found),
-        do: !conn.assigns.remote_ip_found,
-        else: false
-
-    localhost_or_socket and remote_ip_not_found
   end
 
   @inspect_bucket_not_found {:error, :not_found}

lib/pleroma/plugs/remote_ip.ex

@@ -7,8 +7,6 @@ defmodule Pleroma.Plugs.RemoteIp do
   This is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration.
   """
 
-  import Plug.Conn
-
   @behaviour Plug
 
   @headers ~w[
@@ -28,12 +26,11 @@ defmodule Pleroma.Plugs.RemoteIp do
 
   def init(_), do: nil
 
-  def call(%{remote_ip: original_remote_ip} = conn, _) do
+  def call(conn, _) do
     config = Pleroma.Config.get(__MODULE__, [])
 
     if Keyword.get(config, :enabled, false) do
-      %{remote_ip: new_remote_ip} = conn = RemoteIp.call(conn, remote_ip_opts(config))
+      RemoteIp.call(conn, remote_ip_opts(config))
-      assign(conn, :remote_ip_found, original_remote_ip != new_remote_ip)
     else
       conn
     end

lib/pleroma/plugs/uploaded_media.ex

@@ -41,6 +41,7 @@ def call(%{request_path: <<"/", @path, "/", file::binary>>} = conn, opts) do
         conn ->
           conn
       end
+      |> merge_resp_headers([{"content-security-policy", "sandbox"}])
 
     config = Pleroma.Config.get(Pleroma.Upload)
 

lib/pleroma/pool/connections.ex

@@ -0,0 +1,283 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Pool.Connections do
+  use GenServer
+
+  alias Pleroma.Config
+  alias Pleroma.Gun
+
+  require Logger
+
+  @type domain :: String.t()
+  @type conn :: Pleroma.Gun.Conn.t()
+
+  @type t :: %__MODULE__{
+          conns: %{domain() => conn()},
+          opts: keyword()
+        }
+
+  defstruct conns: %{}, opts: []
+
+  @spec start_link({atom(), keyword()}) :: {:ok, pid()}
+  def start_link({name, opts}) do
+    GenServer.start_link(__MODULE__, opts, name: name)
+  end
+
+  @impl true
+  def init(opts), do: {:ok, %__MODULE__{conns: %{}, opts: opts}}
+
+  @spec checkin(String.t() | URI.t(), atom()) :: pid() | nil
+  def checkin(url, name)
+  def checkin(url, name) when is_binary(url), do: checkin(URI.parse(url), name)
+
+  def checkin(%URI{} = uri, name) do
+    timeout = Config.get([:connections_pool, :checkin_timeout], 250)
+
+    GenServer.call(name, {:checkin, uri}, timeout)
+  end
+
+  @spec alive?(atom()) :: boolean()
+  def alive?(name) do
+    if pid = Process.whereis(name) do
+      Process.alive?(pid)
+    else
+      false
+    end
+  end
+
+  @spec get_state(atom()) :: t()
+  def get_state(name) do
+    GenServer.call(name, :state)
+  end
+
+  @spec count(atom()) :: pos_integer()
+  def count(name) do
+    GenServer.call(name, :count)
+  end
+
+  @spec get_unused_conns(atom()) :: [{domain(), conn()}]
+  def get_unused_conns(name) do
+    GenServer.call(name, :unused_conns)
+  end
+
+  @spec checkout(pid(), pid(), atom()) :: :ok
+  def checkout(conn, pid, name) do
+    GenServer.cast(name, {:checkout, conn, pid})
+  end
+
+  @spec add_conn(atom(), String.t(), Pleroma.Gun.Conn.t()) :: :ok
+  def add_conn(name, key, conn) do
+    GenServer.cast(name, {:add_conn, key, conn})
+  end
+
+  @spec remove_conn(atom(), String.t()) :: :ok
+  def remove_conn(name, key) do
+    GenServer.cast(name, {:remove_conn, key})
+  end
+
+  @impl true
+  def handle_cast({:add_conn, key, conn}, state) do
+    state = put_in(state.conns[key], conn)
+
+    Process.monitor(conn.conn)
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_cast({:checkout, conn_pid, pid}, state) do
+    state =
+      with true <- Process.alive?(conn_pid),
+           {key, conn} <- find_conn(state.conns, conn_pid),
+           used_by <- List.keydelete(conn.used_by, pid, 0) do
+        conn_state = if used_by == [], do: :idle, else: conn.conn_state
+
+        put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by})
+      else
+        false ->
+          Logger.debug("checkout for closed conn #{inspect(conn_pid)}")
+          state
+
+        nil ->
+          Logger.debug("checkout for alive conn #{inspect(conn_pid)}, but is not in state")
+          state
+      end
+
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_cast({:remove_conn, key}, state) do
+    state = put_in(state.conns, Map.delete(state.conns, key))
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_call({:checkin, uri}, from, state) do
+    key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
+
+    case state.conns[key] do
+      %{conn: pid, gun_state: :up} = conn ->
+        time = :os.system_time(:second)
+        last_reference = time - conn.last_reference
+        crf = crf(last_reference, 100, conn.crf)
+
+        state =
+          put_in(state.conns[key], %{
+            conn
+            | last_reference: time,
+              crf: crf,
+              conn_state: :active,
+              used_by: [from | conn.used_by]
+          })
+
+        {:reply, pid, state}
+
+      %{gun_state: :down} ->
+        {:reply, nil, state}
+
+      nil ->
+        {:reply, nil, state}
+    end
+  end
+
+  @impl true
+  def handle_call(:state, _from, state), do: {:reply, state, state}
+
+  @impl true
+  def handle_call(:count, _from, state) do
+    {:reply, Enum.count(state.conns), state}
+  end
+
+  @impl true
+  def handle_call(:unused_conns, _from, state) do
+    unused_conns =
+      state.conns
+      |> Enum.filter(&filter_conns/1)
+      |> Enum.sort(&sort_conns/2)
+
+    {:reply, unused_conns, state}
+  end
+
+  defp filter_conns({_, %{conn_state: :idle, used_by: []}}), do: true
+  defp filter_conns(_), do: false
+
+  defp sort_conns({_, c1}, {_, c2}) do
+    c1.crf <= c2.crf and c1.last_reference <= c2.last_reference
+  end
+
+  @impl true
+  def handle_info({:gun_up, conn_pid, _protocol}, state) do
+    %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid)
+
+    host =
+      case :inet.ntoa(host) do
+        {:error, :einval} -> host
+        ip -> ip
+      end
+
+    key = "#{scheme}:#{host}:#{port}"
+
+    state =
+      with {key, conn} <- find_conn(state.conns, conn_pid, key),
+           {true, key} <- {Process.alive?(conn_pid), key} do
+        put_in(state.conns[key], %{
+          conn
+          | gun_state: :up,
+            conn_state: :active,
+            retries: 0
+        })
+      else
+        {false, key} ->
+          put_in(
+            state.conns,
+            Map.delete(state.conns, key)
+          )
+
+        nil ->
+          :ok = Gun.close(conn_pid)
+
+          state
+      end
+
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do
+    retries = Config.get([:connections_pool, :retry], 1)
+    # we can't get info on this pid, because pid is dead
+    state =
+      with {key, conn} <- find_conn(state.conns, conn_pid),
+           {true, key} <- {Process.alive?(conn_pid), key} do
+        if conn.retries == retries do
+          :ok = Gun.close(conn.conn)
+
+          put_in(
+            state.conns,
+            Map.delete(state.conns, key)
+          )
+        else
+          put_in(state.conns[key], %{
+            conn
+            | gun_state: :down,
+              retries: conn.retries + 1
+          })
+        end
+      else
+        {false, key} ->
+          put_in(
+            state.conns,
+            Map.delete(state.conns, key)
+          )
+
+        nil ->
+          Logger.debug(":gun_down for conn which isn't found in state")
+
+          state
+      end
+
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do
+    Logger.debug("received DOWN message for #{inspect(conn_pid)} reason -> #{inspect(reason)}")
+
+    state =
+      with {key, conn} <- find_conn(state.conns, conn_pid) do
+        Enum.each(conn.used_by, fn {pid, _ref} ->
+          Process.exit(pid, reason)
+        end)
+
+        put_in(
+          state.conns,
+          Map.delete(state.conns, key)
+        )
+      else
+        nil ->
+          Logger.debug(":DOWN for conn which isn't found in state")
+
+          state
+      end
+
+    {:noreply, state}
+  end
+
+  defp find_conn(conns, conn_pid) do
+    Enum.find(conns, fn {_key, conn} ->
+      conn.conn == conn_pid
+    end)
+  end
+
+  defp find_conn(conns, conn_pid, conn_key) do
+    Enum.find(conns, fn {key, conn} ->
+      key == conn_key and conn.conn == conn_pid
+    end)
+  end
+
+  def crf(current, steps, crf) do
+    1 + :math.pow(0.5, current / steps) * crf
+  end
+end

lib/pleroma/pool/pool.ex

@@ -0,0 +1,22 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Pool do
+  def child_spec(opts) do
+    poolboy_opts =
+      opts
+      |> Keyword.put(:worker_module, Pleroma.Pool.Request)
+      |> Keyword.put(:name, {:local, opts[:name]})
+      |> Keyword.put(:size, opts[:size])
+      |> Keyword.put(:max_overflow, opts[:max_overflow])
+
+    %{
+      id: opts[:id] || {__MODULE__, make_ref()},
+      start: {:poolboy, :start_link, [poolboy_opts, [name: opts[:name]]]},
+      restart: :permanent,
+      shutdown: 5000,
+      type: :worker
+    }
+  end
+end

lib/pleroma/pool/request.ex

@@ -0,0 +1,65 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Pool.Request do
+  use GenServer
+
+  require Logger
+
+  def start_link(args) do
+    GenServer.start_link(__MODULE__, args)
+  end
+
+  @impl true
+  def init(_), do: {:ok, []}
+
+  @spec execute(pid() | atom(), Tesla.Client.t(), keyword(), pos_integer()) ::
+          {:ok, Tesla.Env.t()} | {:error, any()}
+  def execute(pid, client, request, timeout) do
+    GenServer.call(pid, {:execute, client, request}, timeout)
+  end
+
+  @impl true
+  def handle_call({:execute, client, request}, _from, state) do
+    response = Pleroma.HTTP.request(client, request)
+
+    {:reply, response, state}
+  end
+
+  @impl true
+  def handle_info({:gun_data, _conn, _stream, _, _}, state) do
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info({:gun_up, _conn, _protocol}, state) do
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info({:gun_down, _conn, _protocol, _reason, _killed}, state) do
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info({:gun_error, _conn, _stream, _error}, state) do
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info({:gun_push, _conn, _stream, _new_stream, _method, _uri, _headers}, state) do
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info({:gun_response, _conn, _stream, _, _status, _headers}, state) do
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info(msg, state) do
+    Logger.warn("Received unexpected message #{inspect(__MODULE__)} #{inspect(msg)}")
+    {:noreply, state}
+  end
+end

lib/pleroma/pool/supervisor.ex

@@ -0,0 +1,42 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Pool.Supervisor do
+  use Supervisor
+
+  alias Pleroma.Config
+  alias Pleroma.Pool
+
+  def start_link(args) do
+    Supervisor.start_link(__MODULE__, args, name: __MODULE__)
+  end
+
+  def init(_) do
+    conns_child = %{
+      id: Pool.Connections,
+      start:
+        {Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]}
+    }
+
+    Supervisor.init([conns_child | pools()], strategy: :one_for_one)
+  end
+
+  defp pools do
+    pools = Config.get(:pools)
+
+    pools =
+      if Config.get([Pleroma.Upload, :proxy_remote]) == false do
+        Keyword.delete(pools, :upload)
+      else
+        pools
+      end
+
+    for {pool_name, pool_opts} <- pools do
+      pool_opts
+      |> Keyword.put(:id, {Pool, pool_name})
+      |> Keyword.put(:name, pool_name)
+      |> Pool.child_spec()
+    end
+  end
+end

lib/pleroma/reverse_proxy/client.ex

@@ -3,19 +3,23 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.ReverseProxy.Client do
-  @callback request(atom(), String.t(), [tuple()], String.t(), list()) ::
+  @type status :: pos_integer()
-              {:ok, pos_integer(), [tuple()], reference() | map()}
+  @type header_name :: String.t()
-              | {:ok, pos_integer(), [tuple()]}
+  @type header_value :: String.t()
+  @type headers :: [{header_name(), header_value()}]
+
+  @callback request(atom(), String.t(), headers(), String.t(), list()) ::
+              {:ok, status(), headers(), reference() | map()}
+              | {:ok, status(), headers()}
               | {:ok, reference()}
               | {:error, term()}
 
-  @callback stream_body(reference() | pid() | map()) ::
+  @callback stream_body(map()) :: {:ok, binary(), map()} | :done | {:error, atom() | String.t()}
-              {:ok, binary()} | :done | {:error, String.t()}
 
   @callback close(reference() | pid() | map()) :: :ok
 
-  def request(method, url, headers, "", opts \\ []) do
+  def request(method, url, headers, body \\ "", opts \\ []) do
-    client().request(method, url, headers, "", opts)
+    client().request(method, url, headers, body, opts)
   end
 
   def stream_body(ref), do: client().stream_body(ref)
@@ -23,6 +27,12 @@ def stream_body(ref), do: client().stream_body(ref)
   def close(ref), do: client().close(ref)
 
   defp client do
-    Pleroma.Config.get([Pleroma.ReverseProxy.Client], :hackney)
+    :tesla
+    |> Application.get_env(:adapter)
+    |> client()
   end
+
+  defp client(Tesla.Adapter.Hackney), do: Pleroma.ReverseProxy.Client.Hackney
+  defp client(Tesla.Adapter.Gun), do: Pleroma.ReverseProxy.Client.Tesla
+  defp client(_), do: Pleroma.Config.get!(Pleroma.ReverseProxy.Client)
 end