merge develop

CHANGELOG.md

@@ -24,15 +24,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - **Breaking:** removed `with_move` parameter from notifications timeline.
 
 ### Added
+- ActivityPub: Added support for existing AP ids for instances migrated from Mastodon.
+- Instance: Add `background_image` to configuration and `/api/v1/instance`
 - Instance: Extend `/api/v1/instance` with Pleroma-specific information.
 - NodeInfo: `pleroma:api/v1/notifications:include_types_filter` to the `features` list.
 - NodeInfo: `pleroma_emoji_reactions` to the `features` list.
 - Configuration: `:restrict_unauthenticated` setting, restrict access for unauthenticated users to timelines (public and federate), user profiles and statuses.
 - Configuration: Add `:database_config_whitelist` setting to whitelist settings which can be configured from AdminFE.
+- Configuration: `filename_display_max_length` option to set filename truncate limit, if filename display enabled (0 = no limit).
 - New HTTP adapter [gun](https://github.com/ninenines/gun). Gun adapter requires minimum OTP version of 22.2 otherwise Pleroma won’t start. For hackney OTP update is not required.
 - Mix task to create trusted OAuth App.
 - Notifications: Added `follow_request` notification type.
 - Added `:reject_deletes` group to SimplePolicy
+- MRF (`EmojiStealPolicy`): New MRF Policy which allows to automatically download emojis from remote instances
 <details>
   <summary>API Changes</summary>
 - Mastodon API: Extended `/api/v1/instance`.
@@ -49,12 +53,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Fix follower/blocks import when nicknames starts with @
 - Filtering of push notifications on activities from blocked domains
 - Resolving Peertube accounts with Webfinger
+- `blob:` urls not being allowed by connect-src CSP
 
 ## [Unreleased (patch)]
 
 ### Fixed
 - Healthcheck reporting the number of memory currently used, rather than allocated in total
-- `InsertSkeletonsForDeletedUsers` failing on some instances 
+- `InsertSkeletonsForDeletedUsers` failing on some instances
 
 ## [2.0.3] - 2020-05-02
 

benchmarks/load_testing/activities.ex

@@ -22,8 +22,21 @@ defmodule Pleroma.LoadTesting.Activities do
   @max_concurrency 10
 
   @visibility ~w(public private direct unlisted)
-  @types ~w(simple emoji mentions hell_thread attachment tag like reblog simple_thread remote)
+  @types [
-  @groups ~w(user friends non_friends)
+    :simple,
+    :emoji,
+    :mentions,
+    :hell_thread,
+    :attachment,
+    :tag,
+    :like,
+    :reblog,
+    :simple_thread
+  ]
+  @groups [:friends_local, :friends_remote, :non_friends_local, :non_friends_local]
+  @remote_groups [:friends_remote, :non_friends_remote]
+  @friends_groups [:friends_local, :friends_remote]
+  @non_friends_groups [:non_friends_local, :non_friends_remote]
 
   @spec generate(User.t(), keyword()) :: :ok
   def generate(user, opts \\ []) do
@@ -34,33 +47,24 @@ defmodule Pleroma.LoadTesting.Activities do
 
     opts = Keyword.merge(@defaults, opts)
 
-    friends =
+    users = Users.prepare_users(user, opts)
-      user
-      |> Users.get_users(limit: opts[:friends_used], local: :local, friends?: true)
-      |> Enum.shuffle()
 
-    non_friends =
+    {:ok, _} = Agent.start_link(fn -> users[:non_friends_remote] end, name: :non_friends_remote)
-      user
-      |> Users.get_users(limit: opts[:non_friends_used], local: :local, friends?: false)
-      |> Enum.shuffle()
 
     task_data =
       for visibility <- @visibility,
           type <- @types,
-          group <- @groups,
+          group <- [:user | @groups],
           do: {visibility, type, group}
 
     IO.puts("Starting generating #{opts[:iterations]} iterations of activities...")
 
-    friends_thread = Enum.take(friends, 5)
-    non_friends_thread = Enum.take(friends, 5)
-
     public_long_thread = fn ->
-      generate_long_thread("public", user, friends_thread, non_friends_thread, opts)
+      generate_long_thread("public", users, opts)
     end
 
     private_long_thread = fn ->
-      generate_long_thread("private", user, friends_thread, non_friends_thread, opts)
+      generate_long_thread("private", users, opts)
     end
 
     iterations = opts[:iterations]
@@ -73,10 +77,10 @@ defmodule Pleroma.LoadTesting.Activities do
             i when i == iterations - 2 ->
               spawn(public_long_thread)
               spawn(private_long_thread)
-              generate_activities(user, friends, non_friends, Enum.shuffle(task_data), opts)
+              generate_activities(users, Enum.shuffle(task_data), opts)
 
             _ ->
-              generate_activities(user, friends, non_friends, Enum.shuffle(task_data), opts)
+              generate_activities(users, Enum.shuffle(task_data), opts)
           end
         )
       end)
@@ -123,22 +127,22 @@ defmodule Pleroma.LoadTesting.Activities do
     Enum.each(1..activity_count, fn _ ->
       random = :rand.uniform()
       i = Enum.find_index(intervals, fn {lower, upper} -> lower <= random && upper > random end)
-      CommonAPI.post(Enum.random(users), %{"status" => "a post with the tag #tag_#{i}"})
+      CommonAPI.post(Enum.random(users), %{status: "a post with the tag #tag_#{i}"})
     end)
   end
 
-  defp generate_long_thread(visibility, user, friends, non_friends, _opts) do
+  defp generate_long_thread(visibility, users, _opts) do
     group =
       if visibility == "public",
-        do: "friends",
+        do: :friends_local,
-        else: "user"
+        else: :user
 
     tasks = get_reply_tasks(visibility, group) |> Stream.cycle() |> Enum.take(50)
 
     {:ok, activity} =
-      CommonAPI.post(user, %{
+      CommonAPI.post(users[:user], %{
-        "status" => "Start of #{visibility} long thread",
+        status: "Start of #{visibility} long thread",
-        "visibility" => visibility
+        visibility: visibility
       })
 
     Agent.update(:benchmark_state, fn state ->
@@ -150,31 +154,28 @@ defmodule Pleroma.LoadTesting.Activities do
       Map.put(state, key, activity)
     end)
 
-    acc = {activity.id, ["@" <> user.nickname, "reply to long thread"]}
+    acc = {activity.id, ["@" <> users[:user].nickname, "reply to long thread"]}
-    insert_replies_for_long_thread(tasks, visibility, user, friends, non_friends, acc)
+    insert_replies_for_long_thread(tasks, visibility, users, acc)
     IO.puts("Generating #{visibility} long thread ended\n")
   end
 
-  defp insert_replies_for_long_thread(tasks, visibility, user, friends, non_friends, acc) do
+  defp insert_replies_for_long_thread(tasks, visibility, users, acc) do
     Enum.reduce(tasks, acc, fn
-      "friend", {id, data} ->
+      :user, {id, data} ->
-        friend = Enum.random(friends)
+        user = users[:user]
-        insert_reply(friend, List.delete(data, "@" <> friend.nickname), id, visibility)
-
-      "non_friend", {id, data} ->
-        non_friend = Enum.random(non_friends)
-        insert_reply(non_friend, List.delete(data, "@" <> non_friend.nickname), id, visibility)
-
-      "user", {id, data} ->
         insert_reply(user, List.delete(data, "@" <> user.nickname), id, visibility)
+
+      group, {id, data} ->
+        replier = Enum.random(users[group])
+        insert_reply(replier, List.delete(data, "@" <> replier.nickname), id, visibility)
     end)
   end
 
-  defp generate_activities(user, friends, non_friends, task_data, opts) do
+  defp generate_activities(users, task_data, opts) do
     Task.async_stream(
       task_data,
       fn {visibility, type, group} ->
-        insert_activity(type, visibility, group, user, friends, non_friends, opts)
+        insert_activity(type, visibility, group, users, opts)
       end,
       max_concurrency: @max_concurrency,
       timeout: 30_000
@@ -182,67 +183,104 @@ defmodule Pleroma.LoadTesting.Activities do
     |> Stream.run()
   end
 
-  defp insert_activity("simple", visibility, group, user, friends, non_friends, _opts) do
+  defp insert_local_activity(visibility, group, users, status) do
-    {:ok, _activity} =
+    {:ok, _} =
       group
-      |> get_actor(user, friends, non_friends)
+      |> get_actor(users)
-      |> CommonAPI.post(%{"status" => "Simple status", "visibility" => visibility})
+      |> CommonAPI.post(%{status: status, visibility: visibility})
   end
 
-  defp insert_activity("emoji", visibility, group, user, friends, non_friends, _opts) do
+  defp insert_remote_activity(visibility, group, users, status) do
-    {:ok, _activity} =
+    actor = get_actor(group, users)
-      group
+    {act_data, obj_data} = prepare_activity_data(actor, visibility, users[:user])
-      |> get_actor(user, friends, non_friends)
+    {activity_data, object_data} = other_data(actor, status)
-      |> CommonAPI.post(%{
+
-        "status" => "Simple status with emoji :firefox:",
+    activity_data
-        "visibility" => visibility
+    |> Map.merge(act_data)
-      })
+    |> Map.put("object", Map.merge(object_data, obj_data))
+    |> Pleroma.Web.ActivityPub.ActivityPub.insert(false)
   end
 
-  defp insert_activity("mentions", visibility, group, user, friends, non_friends, _opts) do
+  defp user_mentions(users) do
     user_mentions =
-      get_random_mentions(friends, Enum.random(0..3)) ++
+      Enum.reduce(
-        get_random_mentions(non_friends, Enum.random(0..3))
+        @groups,
+        [],
+        fn group, acc ->
+          acc ++ get_random_mentions(users[group], Enum.random(0..2))
+        end
+      )
 
-    user_mentions =
+    if Enum.random([true, false]),
-      if Enum.random([true, false]),
+      do: ["@" <> users[:user].nickname | user_mentions],
-        do: ["@" <> user.nickname | user_mentions],
+      else: user_mentions
-        else: user_mentions
-
-    {:ok, _activity} =
-      group
-      |> get_actor(user, friends, non_friends)
-      |> CommonAPI.post(%{
-        "status" => Enum.join(user_mentions, ", ") <> " simple status with mentions",
-        "visibility" => visibility
-      })
   end
 
-  defp insert_activity("hell_thread", visibility, group, user, friends, non_friends, _opts) do
+  defp hell_thread_mentions(users) do
-    mentions =
+    with {:ok, nil} <- Cachex.get(:user_cache, "hell_thread_mentions") do
-      with {:ok, nil} <- Cachex.get(:user_cache, "hell_thread_mentions") do
+      cached =
-        cached =
+        @groups
-          ([user | Enum.take(friends, 10)] ++ Enum.take(non_friends, 10))
+        |> Enum.reduce([users[:user]], fn group, acc ->
-          |> Enum.map(&"@#{&1.nickname}")
+          acc ++ Enum.take(users[group], 5)
-          |> Enum.join(", ")
+        end)
+        |> Enum.map(&"@#{&1.nickname}")
+        |> Enum.join(", ")
 
-        Cachex.put(:user_cache, "hell_thread_mentions", cached)
+      Cachex.put(:user_cache, "hell_thread_mentions", cached)
-        cached
+      cached
-      else
+    else
-        {:ok, cached} -> cached
+      {:ok, cached} -> cached
-      end
+    end
-
-    {:ok, _activity} =
-      group
-      |> get_actor(user, friends, non_friends)
-      |> CommonAPI.post(%{
-        "status" => mentions <> " hell thread status",
-        "visibility" => visibility
-      })
   end
 
-  defp insert_activity("attachment", visibility, group, user, friends, non_friends, _opts) do
+  defp insert_activity(:simple, visibility, group, users, _opts)
-    actor = get_actor(group, user, friends, non_friends)
+       when group in @remote_groups do
+    insert_remote_activity(visibility, group, users, "Remote status")
+  end
+
+  defp insert_activity(:simple, visibility, group, users, _opts) do
+    insert_local_activity(visibility, group, users, "Simple status")
+  end
+
+  defp insert_activity(:emoji, visibility, group, users, _opts)
+       when group in @remote_groups do
+    insert_remote_activity(visibility, group, users, "Remote status with emoji :firefox:")
+  end
+
+  defp insert_activity(:emoji, visibility, group, users, _opts) do
+    insert_local_activity(visibility, group, users, "Simple status with emoji :firefox:")
+  end
+
+  defp insert_activity(:mentions, visibility, group, users, _opts)
+       when group in @remote_groups do
+    mentions = user_mentions(users)
+
+    status = Enum.join(mentions, ", ") <> " remote status with mentions"
+
+    insert_remote_activity(visibility, group, users, status)
+  end
+
+  defp insert_activity(:mentions, visibility, group, users, _opts) do
+    mentions = user_mentions(users)
+
+    status = Enum.join(mentions, ", ") <> " simple status with mentions"
+    insert_remote_activity(visibility, group, users, status)
+  end
+
+  defp insert_activity(:hell_thread, visibility, group, users, _)
+       when group in @remote_groups do
+    mentions = hell_thread_mentions(users)
+    insert_remote_activity(visibility, group, users, mentions <> " remote hell thread status")
+  end
+
+  defp insert_activity(:hell_thread, visibility, group, users, _opts) do
+    mentions = hell_thread_mentions(users)
+
+    insert_local_activity(visibility, group, users, mentions <> " hell thread status")
+  end
+
+  defp insert_activity(:attachment, visibility, group, users, _opts) do
+    actor = get_actor(group, users)
 
     obj_data = %{
       "actor" => actor.ap_id,
@@ -262,118 +300,94 @@ defmodule Pleroma.LoadTesting.Activities do
 
     {:ok, _activity} =
       CommonAPI.post(actor, %{
-        "status" => "Post with attachment",
+        status: "Post with attachment",
-        "visibility" => visibility,
+        visibility: visibility,
-        "media_ids" => [object.id]
+        media_ids: [object.id]
       })
   end
 
-  defp insert_activity("tag", visibility, group, user, friends, non_friends, _opts) do
+  defp insert_activity(:tag, visibility, group, users, _opts) do
-    {:ok, _activity} =
+    insert_local_activity(visibility, group, users, "Status with #tag")
-      group
-      |> get_actor(user, friends, non_friends)
-      |> CommonAPI.post(%{"status" => "Status with #tag", "visibility" => visibility})
   end
 
-  defp insert_activity("like", visibility, group, user, friends, non_friends, opts) do
+  defp insert_activity(:like, visibility, group, users, opts) do
-    actor = get_actor(group, user, friends, non_friends)
+    actor = get_actor(group, users)
 
     with activity_id when not is_nil(activity_id) <- get_random_create_activity_id(),
          {:ok, _activity} <- CommonAPI.favorite(actor, activity_id) do
       :ok
     else
       {:error, _} ->
-        insert_activity("like", visibility, group, user, friends, non_friends, opts)
+        insert_activity(:like, visibility, group, users, opts)
 
       nil ->
         Process.sleep(15)
-        insert_activity("like", visibility, group, user, friends, non_friends, opts)
+        insert_activity(:like, visibility, group, users, opts)
     end
   end
 
-  defp insert_activity("reblog", visibility, group, user, friends, non_friends, opts) do
+  defp insert_activity(:reblog, visibility, group, users, opts) do
-    actor = get_actor(group, user, friends, non_friends)
+    actor = get_actor(group, users)
 
     with activity_id when not is_nil(activity_id) <- get_random_create_activity_id(),
-         {:ok, _activity, _object} <- CommonAPI.repeat(activity_id, actor) do
+         {:ok, _activity} <- CommonAPI.repeat(activity_id, actor) do
       :ok
     else
       {:error, _} ->
-        insert_activity("reblog", visibility, group, user, friends, non_friends, opts)
+        insert_activity(:reblog, visibility, group, users, opts)
 
       nil ->
         Process.sleep(15)
-        insert_activity("reblog", visibility, group, user, friends, non_friends, opts)
+        insert_activity(:reblog, visibility, group, users, opts)
     end
   end
 
-  defp insert_activity("simple_thread", visibility, group, user, friends, non_friends, _opts)
+  defp insert_activity(:simple_thread, "direct", group, users, _opts) do
-       when visibility in ["public", "unlisted", "private"] do
+    actor = get_actor(group, users)
-    actor = get_actor(group, user, friends, non_friends)
-    tasks = get_reply_tasks(visibility, group)
-
-    {:ok, activity} =
-      CommonAPI.post(user, %{"status" => "Simple status", "visibility" => visibility})
-
-    acc = {activity.id, ["@" <> actor.nickname, "reply to status"]}
-    insert_replies(tasks, visibility, user, friends, non_friends, acc)
-  end
-
-  defp insert_activity("simple_thread", "direct", group, user, friends, non_friends, _opts) do
-    actor = get_actor(group, user, friends, non_friends)
     tasks = get_reply_tasks("direct", group)
 
     list =
       case group do
-        "non_friends" ->
+        :user ->
-          Enum.take(non_friends, 3)
+          group = Enum.random(@friends_groups)
+          Enum.take(users[group], 3)
 
         _ ->
-          Enum.take(friends, 3)
+          Enum.take(users[group], 3)
       end
 
     data = Enum.map(list, &("@" <> &1.nickname))
 
     {:ok, activity} =
       CommonAPI.post(actor, %{
-        "status" => Enum.join(data, ", ") <> "simple status",
+        status: Enum.join(data, ", ") <> "simple status",
-        "visibility" => "direct"
+        visibility: "direct"
       })
 
-    acc = {activity.id, ["@" <> user.nickname | data] ++ ["reply to status"]}
+    acc = {activity.id, ["@" <> users[:user].nickname | data] ++ ["reply to status"]}
-    insert_direct_replies(tasks, user, list, acc)
+    insert_direct_replies(tasks, users[:user], list, acc)
   end
 
-  defp insert_activity("remote", _, "user", _, _, _, _), do: :ok
+  defp insert_activity(:simple_thread, visibility, group, users, _opts) do
+    actor = get_actor(group, users)
+    tasks = get_reply_tasks(visibility, group)
 
-  defp insert_activity("remote", visibility, group, user, _friends, _non_friends, opts) do
+    {:ok, activity} =
-    remote_friends =
+      CommonAPI.post(users[:user], %{status: "Simple status", visibility: visibility})
-      Users.get_users(user, limit: opts[:friends_used], local: :external, friends?: true)
 
-    remote_non_friends =
+    acc = {activity.id, ["@" <> actor.nickname, "reply to status"]}
-      Users.get_users(user, limit: opts[:non_friends_used], local: :external, friends?: false)
+    insert_replies(tasks, visibility, users, acc)
-
-    actor = get_actor(group, user, remote_friends, remote_non_friends)
-
-    {act_data, obj_data} = prepare_activity_data(actor, visibility, user)
-    {activity_data, object_data} = other_data(actor)
-
-    activity_data
-    |> Map.merge(act_data)
-    |> Map.put("object", Map.merge(object_data, obj_data))
-    |> Pleroma.Web.ActivityPub.ActivityPub.insert(false)
   end
 
-  defp get_actor("user", user, _friends, _non_friends), do: user
+  defp get_actor(:user, %{user: user}), do: user
-  defp get_actor("friends", _user, friends, _non_friends), do: Enum.random(friends)
+  defp get_actor(group, users), do: Enum.random(users[group])
-  defp get_actor("non_friends", _user, _friends, non_friends), do: Enum.random(non_friends)
 
-  defp other_data(actor) do
+  defp other_data(actor, content) do
     %{host: host} = URI.parse(actor.ap_id)
     datetime = DateTime.utc_now()
-    context_id = "http://#{host}:4000/contexts/#{UUID.generate()}"
+    context_id = "https://#{host}/contexts/#{UUID.generate()}"
-    activity_id = "http://#{host}:4000/activities/#{UUID.generate()}"
+    activity_id = "https://#{host}/activities/#{UUID.generate()}"
-    object_id = "http://#{host}:4000/objects/#{UUID.generate()}"
+    object_id = "https://#{host}/objects/#{UUID.generate()}"
 
     activity_data = %{
       "actor" => actor.ap_id,
@@ -390,7 +404,7 @@ defmodule Pleroma.LoadTesting.Activities do
       "attributedTo" => actor.ap_id,
       "bcc" => [],
       "bto" => [],
-      "content" => "Remote post",
+      "content" => content,
       "context" => context_id,
       "conversation" => context_id,
       "emoji" => %{},
@@ -476,60 +490,74 @@ defmodule Pleroma.LoadTesting.Activities do
     {act_data, obj_data}
   end
 
-  defp get_reply_tasks("public", "user"), do: ~w(friend non_friend user)
+  defp get_reply_tasks("public", :user) do
-  defp get_reply_tasks("public", "friends"), do: ~w(non_friend user friend)
+    [:friends_local, :friends_remote, :non_friends_local, :non_friends_remote, :user]
-  defp get_reply_tasks("public", "non_friends"), do: ~w(user friend non_friend)
+  end
 
-  defp get_reply_tasks(visibility, "user") when visibility in ["unlisted", "private"],
+  defp get_reply_tasks("public", group) when group in @friends_groups do
-    do: ~w(friend user friend)
+    [:non_friends_local, :non_friends_remote, :user, :friends_local, :friends_remote]
+  end
 
-  defp get_reply_tasks(visibility, "friends") when visibility in ["unlisted", "private"],
+  defp get_reply_tasks("public", group) when group in @non_friends_groups do
-    do: ~w(user friend user)
+    [:user, :friends_local, :friends_remote, :non_friends_local, :non_friends_remote]
+  end
 
-  defp get_reply_tasks(visibility, "non_friends") when visibility in ["unlisted", "private"],
+  defp get_reply_tasks(visibility, :user) when visibility in ["unlisted", "private"] do
-    do: []
+    [:friends_local, :friends_remote, :user, :friends_local, :friends_remote]
+  end
 
-  defp get_reply_tasks("direct", "user"), do: ~w(friend user friend)
+  defp get_reply_tasks(visibility, group)
-  defp get_reply_tasks("direct", "friends"), do: ~w(user friend user)
+       when visibility in ["unlisted", "private"] and group in @friends_groups do
-  defp get_reply_tasks("direct", "non_friends"), do: ~w(user non_friend user)
+    [:user, :friends_remote, :friends_local, :user]
+  end
 
-  defp insert_replies(tasks, visibility, user, friends, non_friends, acc) do
+  defp get_reply_tasks(visibility, group)
+       when visibility in ["unlisted", "private"] and
+              group in @non_friends_groups,
+       do: []
+
+  defp get_reply_tasks("direct", :user), do: [:friends_local, :user, :friends_remote]
+
+  defp get_reply_tasks("direct", group) when group in @friends_groups,
+    do: [:user, group, :user]
+
+  defp get_reply_tasks("direct", group) when group in @non_friends_groups do
+    [:user, :non_friends_remote, :user, :non_friends_local]
+  end
+
+  defp insert_replies(tasks, visibility, users, acc) do
     Enum.reduce(tasks, acc, fn
-      "friend", {id, data} ->
+      :user, {id, data} ->
-        friend = Enum.random(friends)
+        insert_reply(users[:user], data, id, visibility)
-        insert_reply(friend, data, id, visibility)
 
-      "non_friend", {id, data} ->
+      group, {id, data} ->
-        non_friend = Enum.random(non_friends)
+        replier = Enum.random(users[group])
-        insert_reply(non_friend, data, id, visibility)
+        insert_reply(replier, data, id, visibility)
-
-      "user", {id, data} ->
-        insert_reply(user, data, id, visibility)
     end)
   end
 
   defp insert_direct_replies(tasks, user, list, acc) do
     Enum.reduce(tasks, acc, fn
-      group, {id, data} when group in ["friend", "non_friend"] ->
+      :user, {id, data} ->
+        {reply_id, _} = insert_reply(user, List.delete(data, "@" <> user.nickname), id, "direct")
+        {reply_id, data}
+
+      _, {id, data} ->
         actor = Enum.random(list)
 
         {reply_id, _} =
           insert_reply(actor, List.delete(data, "@" <> actor.nickname), id, "direct")
 
         {reply_id, data}
-
-      "user", {id, data} ->
-        {reply_id, _} = insert_reply(user, List.delete(data, "@" <> user.nickname), id, "direct")
-        {reply_id, data}
     end)
   end
 
   defp insert_reply(actor, data, activity_id, visibility) do
     {:ok, reply} =
       CommonAPI.post(actor, %{
-        "status" => Enum.join(data, ", "),
+        status: Enum.join(data, ", "),
-        "visibility" => visibility,
+        visibility: visibility,
-        "in_reply_to_status_id" => activity_id
+        in_reply_to_status_id: activity_id
       })
 
     {reply.id, ["@" <> actor.nickname | data]}

benchmarks/load_testing/fetcher.ex

@@ -36,6 +36,7 @@ defmodule Pleroma.LoadTesting.Fetcher do
     fetch_home_timeline(user)
     fetch_direct_timeline(user)
     fetch_public_timeline(user)
+    fetch_public_timeline(user, :with_blocks)
     fetch_public_timeline(user, :local)
     fetch_public_timeline(user, :tag)
     fetch_notifications(user)
@@ -51,12 +52,12 @@ defmodule Pleroma.LoadTesting.Fetcher do
 
   defp opts_for_home_timeline(user) do
     %{
-      "blocking_user" => user,
+      blocking_user: user,
-      "count" => "20",
+      count: "20",
-      "muting_user" => user,
+      muting_user: user,
-      "type" => ["Create", "Announce"],
+      type: ["Create", "Announce"],
-      "user" => user,
+      user: user,
-      "with_muted" => "true"
+      with_muted: true
     }
   end
 
@@ -69,17 +70,17 @@ defmodule Pleroma.LoadTesting.Fetcher do
       ActivityPub.fetch_activities(recipients, opts) |> Enum.reverse() |> List.last()
 
     second_page_last =
-      ActivityPub.fetch_activities(recipients, Map.put(opts, "max_id", first_page_last.id))
+      ActivityPub.fetch_activities(recipients, Map.put(opts, :max_id, first_page_last.id))
       |> Enum.reverse()
       |> List.last()
 
     third_page_last =
-      ActivityPub.fetch_activities(recipients, Map.put(opts, "max_id", second_page_last.id))
+      ActivityPub.fetch_activities(recipients, Map.put(opts, :max_id, second_page_last.id))
       |> Enum.reverse()
       |> List.last()
 
     forth_page_last =
-      ActivityPub.fetch_activities(recipients, Map.put(opts, "max_id", third_page_last.id))
+      ActivityPub.fetch_activities(recipients, Map.put(opts, :max_id, third_page_last.id))
       |> Enum.reverse()
       |> List.last()
 
@@ -89,19 +90,19 @@ defmodule Pleroma.LoadTesting.Fetcher do
       },
       inputs: %{
         "1 page" => opts,
-        "2 page" => Map.put(opts, "max_id", first_page_last.id),
+        "2 page" => Map.put(opts, :max_id, first_page_last.id),
-        "3 page" => Map.put(opts, "max_id", second_page_last.id),
+        "3 page" => Map.put(opts, :max_id, second_page_last.id),
-        "4 page" => Map.put(opts, "max_id", third_page_last.id),
+        "4 page" => Map.put(opts, :max_id, third_page_last.id),
-        "5 page" => Map.put(opts, "max_id", forth_page_last.id),
+        "5 page" => Map.put(opts, :max_id, forth_page_last.id),
-        "1 page only media" => Map.put(opts, "only_media", "true"),
+        "1 page only media" => Map.put(opts, :only_media, true),
         "2 page only media" =>
-          Map.put(opts, "max_id", first_page_last.id) |> Map.put("only_media", "true"),
+          Map.put(opts, :max_id, first_page_last.id) |> Map.put(:only_media, true),
         "3 page only media" =>
-          Map.put(opts, "max_id", second_page_last.id) |> Map.put("only_media", "true"),
+          Map.put(opts, :max_id, second_page_last.id) |> Map.put(:only_media, true),
         "4 page only media" =>
-          Map.put(opts, "max_id", third_page_last.id) |> Map.put("only_media", "true"),
+          Map.put(opts, :max_id, third_page_last.id) |> Map.put(:only_media, true),
         "5 page only media" =>
-          Map.put(opts, "max_id", forth_page_last.id) |> Map.put("only_media", "true")
+          Map.put(opts, :max_id, forth_page_last.id) |> Map.put(:only_media, true)
       },
       formatters: formatters()
     )
@@ -109,12 +110,12 @@ defmodule Pleroma.LoadTesting.Fetcher do
 
   defp opts_for_direct_timeline(user) do
     %{
-      :visibility => "direct",
+      visibility: "direct",
-      "blocking_user" => user,
+      blocking_user: user,
-      "count" => "20",
+      count: "20",
-      "type" => "Create",
+      type: "Create",
-      "user" => user,
+      user: user,
-      "with_muted" => "true"
+      with_muted: true
     }
   end
 
@@ -129,7 +130,7 @@ defmodule Pleroma.LoadTesting.Fetcher do
       |> Pagination.fetch_paginated(opts)
       |> List.last()
 
-    opts2 = Map.put(opts, "max_id", first_page_last.id)
+    opts2 = Map.put(opts, :max_id, first_page_last.id)
 
     second_page_last =
       recipients
@@ -137,7 +138,7 @@ defmodule Pleroma.LoadTesting.Fetcher do
       |> Pagination.fetch_paginated(opts2)
       |> List.last()
 
-    opts3 = Map.put(opts, "max_id", second_page_last.id)
+    opts3 = Map.put(opts, :max_id, second_page_last.id)
 
     third_page_last =
       recipients
@@ -145,7 +146,7 @@ defmodule Pleroma.LoadTesting.Fetcher do
       |> Pagination.fetch_paginated(opts3)
       |> List.last()
 
-    opts4 = Map.put(opts, "max_id", third_page_last.id)
+    opts4 = Map.put(opts, :max_id, third_page_last.id)
 
     forth_page_last =
       recipients
@@ -164,7 +165,7 @@ defmodule Pleroma.LoadTesting.Fetcher do
         "2 page" => opts2,
         "3 page" => opts3,
         "4 page" => opts4,
-        "5 page" => Map.put(opts4, "max_id", forth_page_last.id)
+        "5 page" => Map.put(opts4, :max_id, forth_page_last.id)
       },
       formatters: formatters()
     )
@@ -172,34 +173,34 @@ defmodule Pleroma.LoadTesting.Fetcher do
 
   defp opts_for_public_timeline(user) do
     %{
-      "type" => ["Create", "Announce"],
+      type: ["Create", "Announce"],
-      "local_only" => false,
+      local_only: false,
-      "blocking_user" => user,
+      blocking_user: user,
-      "muting_user" => user
+      muting_user: user
     }
   end
 
   defp opts_for_public_timeline(user, :local) do
     %{
-      "type" => ["Create", "Announce"],
+      type: ["Create", "Announce"],
-      "local_only" => true,
+      local_only: true,
-      "blocking_user" => user,
+      blocking_user: user,
-      "muting_user" => user
+      muting_user: user
     }
   end
 
   defp opts_for_public_timeline(user, :tag) do
     %{
-      "blocking_user" => user,
+      blocking_user: user,
-      "count" => "20",
+      count: "20",
-      "local_only" => nil,
+      local_only: nil,
-      "muting_user" => user,
+      muting_user: user,
-      "tag" => ["tag"],
+      tag: ["tag"],
-      "tag_all" => [],
+      tag_all: [],
-      "tag_reject" => [],
+      tag_reject: [],
-      "type" => "Create",
+      type: "Create",
-      "user" => user,
+      user: user,
-      "with_muted" => "true"
+      with_muted: true
     }
   end
 
@@ -222,24 +223,72 @@ defmodule Pleroma.LoadTesting.Fetcher do
   end
 
   defp fetch_public_timeline(user, :only_media) do
-    opts = opts_for_public_timeline(user) |> Map.put("only_media", "true")
+    opts = opts_for_public_timeline(user) |> Map.put(:only_media, true)
 
     fetch_public_timeline(opts, "public timeline only media")
   end
 
+  defp fetch_public_timeline(user, :with_blocks) do
+    opts = opts_for_public_timeline(user)
+
+    remote_non_friends = Agent.get(:non_friends_remote, & &1)
+
+    Benchee.run(%{
+      "public timeline without blocks" => fn ->
+        ActivityPub.fetch_public_activities(opts)
+      end
+    })
+
+    Enum.each(remote_non_friends, fn non_friend ->
+      {:ok, _} = User.block(user, non_friend)
+    end)
+
+    user = User.get_by_id(user.id)
+
+    opts = Map.put(opts, :blocking_user, user)
+
+    Benchee.run(%{
+      "public timeline with user block" => fn ->
+        ActivityPub.fetch_public_activities(opts)
+      end
+    })
+
+    domains =
+      Enum.reduce(remote_non_friends, [], fn non_friend, domains ->
+        {:ok, _user} = User.unblock(user, non_friend)
+        %{host: host} = URI.parse(non_friend.ap_id)
+        [host | domains]
+      end)
+
+    domains = Enum.uniq(domains)
+
+    Enum.each(domains, fn domain ->
+      {:ok, _} = User.block_domain(user, domain)
+    end)
+
+    user = User.get_by_id(user.id)
+    opts = Map.put(opts, :blocking_user, user)
+
+    Benchee.run(%{
+      "public timeline with domain block" => fn ->
+        ActivityPub.fetch_public_activities(opts)
+      end
+    })
+  end
+
   defp fetch_public_timeline(opts, title) when is_binary(title) do
     first_page_last = ActivityPub.fetch_public_activities(opts) |> List.last()
 
     second_page_last =
-      ActivityPub.fetch_public_activities(Map.put(opts, "max_id", first_page_last.id))
+      ActivityPub.fetch_public_activities(Map.put(opts, :max_id, first_page_last.id))
       |> List.last()
 
     third_page_last =
-      ActivityPub.fetch_public_activities(Map.put(opts, "max_id", second_page_last.id))
+      ActivityPub.fetch_public_activities(Map.put(opts, :max_id, second_page_last.id))
       |> List.last()
 
     forth_page_last =
-      ActivityPub.fetch_public_activities(Map.put(opts, "max_id", third_page_last.id))
+      ActivityPub.fetch_public_activities(Map.put(opts, :max_id, third_page_last.id))
       |> List.last()
 
     Benchee.run(
@@ -250,17 +299,17 @@ defmodule Pleroma.LoadTesting.Fetcher do
       },
       inputs: %{
         "1 page" => opts,
-        "2 page" => Map.put(opts, "max_id", first_page_last.id),
+        "2 page" => Map.put(opts, :max_id, first_page_last.id),
-        "3 page" => Map.put(opts, "max_id", second_page_last.id),
+        "3 page" => Map.put(opts, :max_id, second_page_last.id),
-        "4 page" => Map.put(opts, "max_id", third_page_last.id),
+        "4 page" => Map.put(opts, :max_id, third_page_last.id),
-        "5 page" => Map.put(opts, "max_id", forth_page_last.id)
+        "5 page" => Map.put(opts, :max_id, forth_page_last.id)
       },
       formatters: formatters()
     )
   end
 
   defp opts_for_notifications do
-    %{"count" => "20", "with_muted" => "true"}
+    %{count: "20", with_muted: true}
   end
 
   defp fetch_notifications(user) do
@@ -269,15 +318,15 @@ defmodule Pleroma.LoadTesting.Fetcher do
     first_page_last = MastodonAPI.get_notifications(user, opts) |> List.last()
 
     second_page_last =
-      MastodonAPI.get_notifications(user, Map.put(opts, "max_id", first_page_last.id))
+      MastodonAPI.get_notifications(user, Map.put(opts, :max_id, first_page_last.id))
       |> List.last()
 
     third_page_last =
-      MastodonAPI.get_notifications(user, Map.put(opts, "max_id", second_page_last.id))
+      MastodonAPI.get_notifications(user, Map.put(opts, :max_id, second_page_last.id))
       |> List.last()
 
     forth_page_last =
-      MastodonAPI.get_notifications(user, Map.put(opts, "max_id", third_page_last.id))
+      MastodonAPI.get_notifications(user, Map.put(opts, :max_id, third_page_last.id))
       |> List.last()
 
     Benchee.run(
@@ -288,10 +337,10 @@ defmodule Pleroma.LoadTesting.Fetcher do
       },
       inputs: %{
         "1 page" => opts,
-        "2 page" => Map.put(opts, "max_id", first_page_last.id),
+        "2 page" => Map.put(opts, :max_id, first_page_last.id),
-        "3 page" => Map.put(opts, "max_id", second_page_last.id),
+        "3 page" => Map.put(opts, :max_id, second_page_last.id),
-        "4 page" => Map.put(opts, "max_id", third_page_last.id),
+        "4 page" => Map.put(opts, :max_id, third_page_last.id),
-        "5 page" => Map.put(opts, "max_id", forth_page_last.id)
+        "5 page" => Map.put(opts, :max_id, forth_page_last.id)
       },
       formatters: formatters()
     )
@@ -301,13 +350,13 @@ defmodule Pleroma.LoadTesting.Fetcher do
     first_page_last = ActivityPub.fetch_favourites(user) |> List.last()
 
     second_page_last =
-      ActivityPub.fetch_favourites(user, %{"max_id" => first_page_last.id}) |> List.last()
+      ActivityPub.fetch_favourites(user, %{:max_id => first_page_last.id}) |> List.last()
 
     third_page_last =
-      ActivityPub.fetch_favourites(user, %{"max_id" => second_page_last.id}) |> List.last()
+      ActivityPub.fetch_favourites(user, %{:max_id => second_page_last.id}) |> List.last()
 
     forth_page_last =
-      ActivityPub.fetch_favourites(user, %{"max_id" => third_page_last.id}) |> List.last()
+      ActivityPub.fetch_favourites(user, %{:max_id => third_page_last.id}) |> List.last()
 
     Benchee.run(
       %{
@@ -317,10 +366,10 @@ defmodule Pleroma.LoadTesting.Fetcher do
       },
       inputs: %{
         "1 page" => %{},
-        "2 page" => %{"max_id" => first_page_last.id},
+        "2 page" => %{:max_id => first_page_last.id},
-        "3 page" => %{"max_id" => second_page_last.id},
+        "3 page" => %{:max_id => second_page_last.id},
-        "4 page" => %{"max_id" => third_page_last.id},
+        "4 page" => %{:max_id => third_page_last.id},
-        "5 page" => %{"max_id" => forth_page_last.id}
+        "5 page" => %{:max_id => forth_page_last.id}
       },
       formatters: formatters()
     )
@@ -328,8 +377,8 @@ defmodule Pleroma.LoadTesting.Fetcher do
 
   defp opts_for_long_thread(user) do
     %{
-      "blocking_user" => user,
+      blocking_user: user,
-      "user" => user
+      user: user
     }
   end
 
@@ -339,9 +388,9 @@ defmodule Pleroma.LoadTesting.Fetcher do
 
     opts = opts_for_long_thread(user)
 
-    private_input = {private.data["context"], Map.put(opts, "exclude_id", private.id)}
+    private_input = {private.data["context"], Map.put(opts, :exclude_id, private.id)}
 
-    public_input = {public.data["context"], Map.put(opts, "exclude_id", public.id)}
+    public_input = {public.data["context"], Map.put(opts, :exclude_id, public.id)}
 
     Benchee.run(
       %{
@@ -461,13 +510,13 @@ defmodule Pleroma.LoadTesting.Fetcher do
     public_context =
       ActivityPub.fetch_activities_for_context(
         public.data["context"],
-        Map.put(fetch_opts, "exclude_id", public.id)
+        Map.put(fetch_opts, :exclude_id, public.id)
       )
 
     private_context =
       ActivityPub.fetch_activities_for_context(
         private.data["context"],
-        Map.put(fetch_opts, "exclude_id", private.id)
+        Map.put(fetch_opts, :exclude_id, private.id)
       )
 
     Benchee.run(
@@ -498,14 +547,14 @@ defmodule Pleroma.LoadTesting.Fetcher do
         end,
         "Public timeline with reply filtering - following" => fn ->
           public_params
-          |> Map.put("reply_visibility", "following")
+          |> Map.put(:reply_visibility, "following")
-          |> Map.put("reply_filtering_user", user)
+          |> Map.put(:reply_filtering_user, user)
           |> ActivityPub.fetch_public_activities()
         end,
         "Public timeline with reply filtering - self" => fn ->
           public_params
-          |> Map.put("reply_visibility", "self")
+          |> Map.put(:reply_visibility, "self")
-          |> Map.put("reply_filtering_user", user)
+          |> Map.put(:reply_filtering_user, user)
           |> ActivityPub.fetch_public_activities()
         end
       },
@@ -524,16 +573,16 @@ defmodule Pleroma.LoadTesting.Fetcher do
         "Home timeline with reply filtering - following" => fn ->
           private_params =
             private_params
-            |> Map.put("reply_filtering_user", user)
+            |> Map.put(:reply_filtering_user, user)
-            |> Map.put("reply_visibility", "following")
+            |> Map.put(:reply_visibility, "following")
 
           ActivityPub.fetch_activities(recipients, private_params)
         end,
         "Home timeline with reply filtering - self" => fn ->
           private_params =
             private_params
-            |> Map.put("reply_filtering_user", user)
+            |> Map.put(:reply_filtering_user, user)
-            |> Map.put("reply_visibility", "self")
+            |> Map.put(:reply_visibility, "self")
 
           ActivityPub.fetch_activities(recipients, private_params)
         end

benchmarks/load_testing/users.ex

@@ -27,7 +27,7 @@ defmodule Pleroma.LoadTesting.Users do
 
     make_friends(main_user, opts[:friends])
 
-    Repo.get(User, main_user.id)
+    User.get_by_id(main_user.id)
   end
 
   def generate_users(max) do
@@ -166,4 +166,24 @@ defmodule Pleroma.LoadTesting.Users do
     )
     |> Stream.run()
   end
+
+  @spec prepare_users(User.t(), keyword()) :: map()
+  def prepare_users(user, opts) do
+    friends_limit = opts[:friends_used]
+    non_friends_limit = opts[:non_friends_used]
+
+    %{
+      user: user,
+      friends_local: fetch_users(user, friends_limit, :local, true),
+      friends_remote: fetch_users(user, friends_limit, :external, true),
+      non_friends_local: fetch_users(user, non_friends_limit, :local, false),
+      non_friends_remote: fetch_users(user, non_friends_limit, :external, false)
+    }
+  end
+
+  defp fetch_users(user, limit, local, friends?) do
+    user
+    |> get_users(limit: limit, local: local, friends?: friends?)
+    |> Enum.shuffle()
+  end
 end

benchmarks/mix/tasks/pleroma/benchmarks/tags.ex

@@ -5,7 +5,6 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
   import Ecto.Query
 
   alias Pleroma.Repo
-  alias Pleroma.Web.MastodonAPI.TimelineController
 
   def run(_args) do
     Mix.Pleroma.start_pleroma()
@@ -37,7 +36,7 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
     Benchee.run(
       %{
         "Hashtag fetching, any" => fn tags ->
-          TimelineController.hashtag_fetching(
+          hashtag_fetching(
             %{
               "any" => tags
             },
@@ -47,7 +46,7 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
         end,
         # Will always return zero results because no overlapping hashtags are generated.
         "Hashtag fetching, all" => fn tags ->
-          TimelineController.hashtag_fetching(
+          hashtag_fetching(
             %{
               "all" => tags
             },
@@ -67,7 +66,7 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
     Benchee.run(
       %{
         "Hashtag fetching" => fn tag ->
-          TimelineController.hashtag_fetching(
+          hashtag_fetching(
             %{
               "tag" => tag
             },
@@ -80,4 +79,35 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
       time: 5
     )
   end
+
+  defp hashtag_fetching(params, user, local_only) do
+    tags =
+      [params["tag"], params["any"]]
+      |> List.flatten()
+      |> Enum.uniq()
+      |> Enum.filter(& &1)
+      |> Enum.map(&String.downcase(&1))
+
+    tag_all =
+      params
+      |> Map.get("all", [])
+      |> Enum.map(&String.downcase(&1))
+
+    tag_reject =
+      params
+      |> Map.get("none", [])
+      |> Enum.map(&String.downcase(&1))
+
+    _activities =
+      params
+      |> Map.put(:type, "Create")
+      |> Map.put(:local_only, local_only)
+      |> Map.put(:blocking_user, user)
+      |> Map.put(:muting_user, user)
+      |> Map.put(:user, user)
+      |> Map.put(:tag, tags)
+      |> Map.put(:tag_all, tag_all)
+      |> Map.put(:tag_reject, tag_reject)
+      |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities()
+  end
 end

config/config.exs

@@ -71,7 +71,8 @@ config :pleroma, Pleroma.Upload,
       follow_redirect: true,
       pool: :upload
     ]
-  ]
+  ],
+  filename_display_max_length: 30
 
 config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads"
 
@@ -170,7 +171,8 @@ config :mime, :types, %{
   "application/ld+json" => ["activity+json"]
 }
 
-config :tesla, adapter: Tesla.Adapter.Gun
+config :tesla, adapter: Tesla.Adapter.Hackney
+
 # Configures http settings, upstream proxy etc.
 config :pleroma, :http,
   proxy_url: nil,
@@ -182,7 +184,8 @@ config :pleroma, :instance,
   name: "Pleroma",
   email: "example@example.com",
   notify_email: "noreply@example.com",
-  description: "A Pleroma instance, an alternative fediverse server",
+  description: "Pleroma: An efficient and flexible fediverse server",
+  background_image: "/images/city.jpg",
   limit: 5_000,
   chat_limit: 5_000,
   remote_limit: 100_000,
@@ -271,20 +274,33 @@ config :pleroma, :markup,
 
 config :pleroma, :frontend_configurations,
   pleroma_fe: %{
-    theme: "pleroma-dark",
+    alwaysShowSubjectInput: true,
-    logo: "/static/logo.png",
     background: "/images/city.jpg",
-    redirectRootNoLogin: "/main/all",
-    redirectRootLogin: "/main/friends",
-    showInstanceSpecificPanel: true,
-    scopeOptionsEnabled: false,
-    formattingOptionsEnabled: false,
     collapseMessageWithSubject: false,
+    disableChat: false,
+    greentext: false,
+    hideFilteredStatuses: false,
+    hideMutedPosts: false,
     hidePostStats: false,
+    hideSitename: false,
     hideUserStats: false,
+    loginMethod: "password",
+    logo: "/static/logo.png",
+    logoMargin: ".1em",
+    logoMask: true,
+    minimalScopesMode: false,
+    noAttachmentLinks: false,
+    nsfwCensorImage: "",
+    postContentType: "text/plain",
+    redirectRootLogin: "/main/friends",
+    redirectRootNoLogin: "/main/all",
     scopeCopy: true,
+    sidebarRight: false,
+    showFeaturesPanel: true,
+    showInstanceSpecificPanel: false,
     subjectLineBehavior: "email",
-    alwaysShowSubjectInput: true
+    theme: "pleroma-dark",
+    webPushNotifications: false
   },
   masto_fe: %{
     showInstanceSpecificPanel: true
@@ -376,6 +392,10 @@ config :pleroma, :rich_media,
 
 config :pleroma, :media_proxy,
   enabled: false,
+  invalidation: [
+    enabled: false,
+    provider: Pleroma.Web.MediaProxy.Invalidation.Script
+  ],
   proxy_opts: [
     redirect_on_failure: false,
     max_body_length: 25 * 1_048_576,

config/description.exs

@@ -119,6 +119,11 @@ config :pleroma, :config_description, [
             ]
           }
         ]
+      },
+      %{
+        key: :filename_display_max_length,
+        type: :integer,
+        description: "Set max length of a filename to display. 0 = no limit. Default: 30"
       }
     ]
   },
@@ -969,6 +974,13 @@ config :pleroma, :config_description, [
             ]
           }
         ]
+      },
+      %{
+        key: :instance_thumbnail,
+        type: :string,
+        description:
+          "The instance thumbnail image. It will appear in [Pleroma Instances](http://distsn.org/pleroma-instances.html)",
+        suggestions: ["/instance/thumbnail.jpeg"]
       }
     ]
   },
@@ -1112,11 +1124,12 @@ config :pleroma, :config_description, [
             logoMask: true,
             minimalScopesMode: false,
             noAttachmentLinks: false,
-            nsfwCensorImage: "",
+            nsfwCensorImage: "/static/img/nsfw.74818f9.png",
             postContentType: "text/plain",
             redirectRootLogin: "/main/friends",
             redirectRootNoLogin: "/main/all",
             scopeCopy: true,
+            sidebarRight: false,
             showFeaturesPanel: true,
             showInstanceSpecificPanel: false,
             subjectLineBehavior: "email",
@@ -1225,7 +1238,7 @@ config :pleroma, :config_description, [
             type: :string,
             description:
               "URL of the image to use for hiding NSFW media attachments in the timeline.",
-            suggestions: ["/static/img/nsfw.png"]
+            suggestions: ["/static/img/nsfw.74818f9.png"]
           },
           %{
             key: :postContentType,
@@ -1256,6 +1269,12 @@ config :pleroma, :config_description, [
             type: :boolean,
             description: "Copy the scope (private/unlisted/public) in replies to posts by default"
           },
+          %{
+            key: :sidebarRight,
+            label: "Sidebar on Right",
+            type: :boolean,
+            description: "Change alignment of sidebar and panels to the right."
+          },
           %{
             key: :showFeaturesPanel,
             label: "Show instance features panel",
@@ -1339,6 +1358,12 @@ config :pleroma, :config_description, [
         suggestions: [
           :pleroma_fox_tan
         ]
+      },
+      %{
+        key: :default_user_avatar,
+        type: :string,
+        description: "URL of the default user avatar.",
+        suggestions: ["/images/avi.png"]
       }
     ]
   },

docs/API/admin_api.md

@@ -511,7 +511,23 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   - `discoverable`
   - `actor_type`
 
-- Response: none (code `200`)
+- Response:
+
+```json
+{"status": "success"}
+```
+
+```json
+{"errors":
+  {"actor_type": "is invalid"},
+  {"email": "has invalid format"},
+  ...
+ }
+```
+
+```json
+{"error": "Unable to update user."}
+```
 
 ## `GET /api/pleroma/admin/reports`
 
@@ -531,7 +547,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 
 ```json
 {
-  "totalReports" : 1,
+  "total" : 1,
   "reports": [
     {
       "account": {
@@ -752,7 +768,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
     - 400 Bad Request `"Invalid parameters"` when `status` is missing
   - On success: `204`, empty response
 
-## `POST /api/pleroma/admin/reports/:report_id/notes/:id`
+## `DELETE /api/pleroma/admin/reports/:report_id/notes/:id`
 
 ### Delete report note
 

docs/API/differences_in_mastoapi_responses.md

@@ -6,10 +6,6 @@ A Pleroma instance can be identified by "<Mastodon version> (compatible; Pleroma
 
 Pleroma uses 128-bit ids as opposed to Mastodon's 64 bits. However just like Mastodon's ids they are lexically sortable strings
 
-## Attachment cap
-
-Some apps operate under the assumption that no more than 4 attachments can be returned or uploaded. Pleroma however does not enforce any limits on attachment count neither when returning the status object nor when posting.
-
 ## Timelines
 
 Adding the parameter `with_muted=true` to the timeline queries will also return activities by muted (not by blocked!) users.
@@ -32,12 +28,20 @@ Has these additional fields under the `pleroma` object:
 - `thread_muted`: true if the thread the post belongs to is muted
 - `emoji_reactions`: A list with emoji / reaction maps. The format is `{name: "☕", count: 1, me: true}`. Contains no information about the reacting users, for that use the `/statuses/:id/reactions` endpoint.
 
-## Attachments
+## Media Attachments
 
 Has these additional fields under the `pleroma` object:
 
 - `mime_type`: mime type of the attachment.
 
+### Attachment cap
+
+Some apps operate under the assumption that no more than 4 attachments can be returned or uploaded. Pleroma however does not enforce any limits on attachment count neither when returning the status object nor when posting.
+
+### Limitations
+
+Pleroma does not process remote images and therefore cannot include fields such as `meta` and `blurhash`. It does not support focal points or aspect ratios. The frontend is expected to handle it.
+
 ## Accounts
 
 The `id` parameter can also be the `nickname` of the user. This only works in these endpoints, not the deeper nested ones for following etc.
@@ -216,6 +220,7 @@ Has theses additional parameters (which are the same as in Pleroma-API):
 - `avatar_upload_limit`: The same for avatars
 - `background_upload_limit`: The same for backgrounds
 - `banner_upload_limit`: The same for banners
+- `background_image`: A background image that frontends can use
 - `pleroma.metadata.features`: A list of supported features
 - `pleroma.metadata.federation`: The federation restrictions of this instance
 - `vapid_public_key`: The public key needed for push messages

docs/API/pleroma_api.md

@@ -265,7 +265,7 @@ See [Admin-API](admin_api.md)
 * Method `PUT`
 * Authentication: required
 * Params:
-    * `image`: Multipart image
+    * `file`: Multipart image
 * Response: JSON. Returns a mastodon media attachment entity
   when successful, otherwise returns HTTP 415 `{"error": "error_msg"}`
 * Example response:
@@ -358,7 +358,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
     * `recipients`: A list of ids of users that should receive posts to this conversation. This will replace the current list of recipients, so submit the full list. The owner of owner of the conversation will always be part of the set of recipients, though.
 * Response: JSON, statuses (200 - healthy, 503 unhealthy)
 
-## `GET /api/v1/pleroma/conversations/read`
+## `POST /api/v1/pleroma/conversations/read`
 ### Marks all user's conversations as read.
 * Method `POST`
 * Authentication: required
@@ -426,7 +426,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 * Authentication: required
 * Params:
   * `file`: file needs to be uploaded with the multipart request or link to remote file.
-  * `shortcode`: (*optional*) shortcode for new emoji, must be uniq for all emoji. If not sended, shortcode will be taken from original filename.
+  * `shortcode`: (*optional*) shortcode for new emoji, must be unique for all emoji. If not sended, shortcode will be taken from original filename.
   * `filename`: (*optional*) new emoji file name. If not specified will be taken from original filename.
 * Response: JSON, list of files for updated pack (hashmap -> shortcode => filename) with status 200, either error status with error message.
 
@@ -536,7 +536,7 @@ Emoji reactions work a lot like favourites do. They make it possible to react to
 ```
 
 ## `GET /api/v1/pleroma/statuses/:id/reactions/:emoji`
-### Get an object of emoji to account mappings with accounts that reacted to the post for a specific emoji`
+### Get an object of emoji to account mappings with accounts that reacted to the post for a specific emoji
 * Method: `GET`
 * Authentication: optional
 * Params: None

docs/administration/CLI_tasks/database.md

@@ -69,3 +69,32 @@ mix pleroma.database update_users_following_followers_counts
 ```sh tab="From Source"
 mix pleroma.database fix_likes_collections
 ```
+
+## Vacuum the database
+
+### Analyze
+
+Running an `analyze` vacuum job can improve performance by updating statistics used by the query planner. **It is safe to cancel this.**
+
+```sh tab="OTP"
+./bin/pleroma_ctl database vacuum analyze
+```
+
+```sh tab="From Source"
+mix pleroma.database vacuum analyze
+```
+
+### Full
+
+Running a `full` vacuum job rebuilds your entire database by reading all of the data and rewriting it into smaller
+and more compact files with an optimized layout. This process will take a long time and use additional disk space as
+it builds the files side-by-side the existing database files. It can make your database faster and use less disk space,
+but should only be run if necessary. **It is safe to cancel this.**
+
+```sh tab="OTP"
+./bin/pleroma_ctl database vacuum full
+```
+
+```sh tab="From Source"
+mix pleroma.database vacuum full
+```

docs/administration/CLI_tasks/user.md

@@ -95,33 +95,33 @@ mix pleroma.user sign_out <nickname>
 ```
 
 
-## Deactivate or activate a user 
+## Deactivate or activate a user
 ```sh tab="OTP"
- ./bin/pleroma_ctl user toggle_activated <nickname> 
+ ./bin/pleroma_ctl user toggle_activated <nickname>
 ```
 
 ```sh tab="From Source"
-mix pleroma.user toggle_activated <nickname> 
+mix pleroma.user toggle_activated <nickname>
 ```
 
 
-## Unsubscribe local users from a user and deactivate the user
+## Deactivate a user and unsubscribes local users from the user
 ```sh tab="OTP"
- ./bin/pleroma_ctl user unsubscribe NICKNAME
+ ./bin/pleroma_ctl user deactivate NICKNAME
 ```
 
 ```sh tab="From Source"
-mix pleroma.user unsubscribe NICKNAME
+mix pleroma.user deactivate NICKNAME
 ```
 
 
-## Unsubscribe local users from an instance and deactivate all accounts on it
+## Deactivate all accounts from an instance and unsubscribe local users on it
 ```sh tab="OTP"
- ./bin/pleroma_ctl user unsubscribe_all_from_instance <instance>
+ ./bin/pleroma_ctl user deactivate_all_from_instance <instance>
 ```
 
 ```sh tab="From Source"
-mix pleroma.user unsubscribe_all_from_instance <instance>
+mix pleroma.user deactivate_all_from_instance <instance>
 ```
 
 
@@ -177,4 +177,3 @@ mix pleroma.user untag <nickname> <tags>
 ```sh tab="From Source"
 mix pleroma.user toggle_confirmed <nickname>
 ```
-

docs/clients.md

@@ -42,6 +42,12 @@ Feel free to contact us to be added to this list!
 - Platforms: SailfishOS
 - Features: No Streaming
 
+### Husky
+- Source code: <https://git.mentality.rip/FWGS/Husky>
+- Contact: [@Husky@enigmatic.observer](https://enigmatic.observer/users/Husky)
+- Platforms: Android
+- Features: No Streaming, Emoji Reactions, Text Formatting, FE Stickers
+
 ### Nekonium
 - Homepage: [F-Droid Repository](https://repo.gdgd.jp.net/), [Google Play](https://play.google.com/store/apps/details?id=com.apps.nekonium), [Amazon](https://www.amazon.co.jp/dp/B076FXPRBC/)
 - Source: <https://gogs.gdgd.jp.net/lin/nekonium>

docs/configuration/cheatsheet.md

@@ -149,6 +149,11 @@ config :pleroma, :mrf_user_allowlist,
   * `:strip_followers` removes followers from the ActivityPub recipient list, ensuring they won't be delivered to home timelines
   * `:reject` rejects the message entirely
 
+#### mrf_steal_emoji
+* `hosts`: List of hosts to steal emojis from
+* `rejected_shortcodes`: Regex-list of shortcodes to reject
+* `size_limit`: File size limit (in bytes), checked before an emoji is saved to the disk
+
 ### :activitypub
 * `unfollow_blocked`: Whether blocks result in people getting unfollowed
 * `outgoing_blocks`: Whether to federate blocks to other instances
@@ -249,6 +254,40 @@ This section describe PWA manifest instance-specific values. Currently this opti
 * `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts.
 * `proxy_opts`: All options defined in `Pleroma.ReverseProxy` documentation, defaults to `[max_body_length: (25*1_048_576)]`.
 * `whitelist`: List of domains to bypass the mediaproxy
+* `invalidation`: options for remove media from cache after delete object:
+    * `enabled`: Enables purge cache
+    * `provider`: Which one of  the [purge cache strategy](#purge-cache-strategy) to use.
+
+### Purge cache strategy
+
+#### Pleroma.Web.MediaProxy.Invalidation.Script
+
+This strategy allow perform external bash script to purge cache.
+Urls of attachments pass to script as arguments.
+
+* `script_path`: path to external script.
+
+Example:
+```elixir
+config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script,
+  script_path: "./installation/nginx-cache-purge.example"
+```
+
+#### Pleroma.Web.MediaProxy.Invalidation.Http
+
+This strategy allow perform custom http request to purge cache.
+
+* `method`: http method. default is `purge`
+* `headers`: http headers. default is empty
+* `options`: request options. default is empty
+
+Example:
+```elixir
+config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Http,
+  method: :purge,
+  headers: [],
+  options: []
+```
 
 ## Link previews
 
@@ -459,6 +498,7 @@ the source code is here: https://github.com/koto-bank/kocaptcha. The default end
 * `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host.
 * `proxy_remote`: If you're using a remote uploader, Pleroma will proxy media requests instead of redirecting to it.
 * `proxy_opts`: Proxy options, see `Pleroma.ReverseProxy` documentation.
+* `filename_display_max_length`: Set max length of a filename to display. 0 = no limit. Default: 30.
 
 !!! warning
     `strip_exif` has been replaced by `Pleroma.Upload.Filter.Mogrify`.
@@ -619,24 +659,6 @@ config :pleroma, :workers,
 * `enabled: false` corresponds to `config :pleroma, :workers, retries: [federator_outgoing: 1]`
 * deprecated options: `max_jobs`, `initial_timeout`
 
-### Pleroma.Scheduler
-
-Configuration for [Quantum](https://github.com/quantum-elixir/quantum-core) jobs scheduler.
-
-See [Quantum readme](https://github.com/quantum-elixir/quantum-core#usage) for the list of supported options.
-
-Example:
-
-```elixir
-config :pleroma, Pleroma.Scheduler,
-  global: true,
-  overlap: true,
-  timezone: :utc,
-  jobs: [{"0 */6 * * * *", {Pleroma.Web.Websub, :refresh_subscriptions, []}}]
-```
-
-The above example defines a single job which invokes `Pleroma.Web.Websub.refresh_subscriptions()` every 6 hours ("0 */6 * * * *", [crontab format](https://en.wikipedia.org/wiki/Cron)).
-
 ## :web_push_encryption, :vapid_details
 
 Web Push Notifications configuration. You can use the mix task `mix web_push.gen.keypair` to generate it.

docs/configuration/postgresql.md

@@ -0,0 +1,31 @@
+# Optimizing your PostgreSQL performance
+
+Pleroma performance depends to a large extent on good database performance. The default PostgreSQL settings are mostly fine, but often you can get better performance by changing a few settings.
+
+You can use [PGTune](https://pgtune.leopard.in.ua) to get recommendations for your setup. If you do, set the "Number of Connections" field to 20, as Pleroma will only use 10 concurrent connections anyway. If you don't, it will give you advice that might even hurt your performance.
+
+We also recommend not using the "Network Storage" option.
+
+## Example configurations
+
+Here are some configuration suggestions for PostgreSQL 10+.
+
+### 1GB RAM, 1 CPU
+```
+shared_buffers = 256MB
+effective_cache_size = 768MB
+maintenance_work_mem = 64MB
+work_mem = 13107kB
+```
+
+### 2GB RAM, 2 CPU
+```
+shared_buffers = 512MB
+effective_cache_size = 1536MB
+maintenance_work_mem = 128MB
+work_mem = 26214kB
+max_worker_processes = 2
+max_parallel_workers_per_gather = 1
+max_parallel_workers = 2
+```
+

docs/configuration/storing_remote_media.md

@@ -0,0 +1,38 @@
+# Storing Remote Media
+
+Pleroma does not store remote/federated media by default. The best way to achieve this is to change Nginx to keep its reverse proxy cache
+for a year and to activate the `MediaProxyWarmingPolicy` MRF policy in Pleroma which will automatically fetch all media through the proxy
+as soon as the post is received by your instance.
+
+## Nginx
+
+```
+    proxy_cache_path /long/term/storage/path/pleroma-media-cache levels=1:2
+        keys_zone=pleroma_media_cache:10m inactive=1y use_temp_path=off;
+
+    location ~ ^/(media|proxy) {
+        proxy_cache        pleroma_media_cache;
+        slice              1m;
+        proxy_cache_key    $host$uri$is_args$args$slice_range;
+        proxy_set_header   Range $slice_range;
+        proxy_http_version 1.1;
+        proxy_cache_valid  206 301 302 304 1h;
+        proxy_cache_valid  200 1y;
+        proxy_cache_use_stale error timeout invalid_header updating;
+        proxy_ignore_client_abort on;
+        proxy_buffering    on;
+        chunked_transfer_encoding on;
+        proxy_ignore_headers Cache-Control Expires;
+        proxy_hide_header  Cache-Control Expires;
+        proxy_pass         http://127.0.0.1:4000;
+    }
+```
+
+## Pleroma
+
+Add to your `prod.secret.exs`:
+
+```
+config :pleroma, :instance,
+  rewrite_policy: [Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy]
+```

docs/installation/debian_based_en.md

@@ -38,8 +38,8 @@ sudo apt install git build-essential postgresql postgresql-contrib
 * Download and add the Erlang repository:
 
 ```shell
-wget -P /tmp/ https://packages.erlang-solutions.com/erlang-solutions_1.0_all.deb
+wget -P /tmp/ https://packages.erlang-solutions.com/erlang-solutions_2.0_all.deb
-sudo dpkg -i /tmp/erlang-solutions_1.0_all.deb
+sudo dpkg -i /tmp/erlang-solutions_2.0_all.deb
 ```
 
 * Install Elixir and Erlang:

docs/installation/debian_based_jp.md

@@ -40,8 +40,8 @@ sudo apt install git build-essential postgresql postgresql-contrib
 
 * Erlangのリポジトリをダウンロードおよびインストールします。
 ```
-wget -P /tmp/ https://packages.erlang-solutions.com/erlang-solutions_1.0_all.deb
+wget -P /tmp/ https://packages.erlang-solutions.com/erlang-solutions_2.0_all.deb
-sudo dpkg -i /tmp/erlang-solutions_1.0_all.deb
+sudo dpkg -i /tmp/erlang-solutions_2.0_all.deb
 ```
 
 * ElixirとErlangをインストールします、

docs/installation/otp_en.md

@@ -63,7 +63,7 @@ apt install postgresql-11-rum
 ```
 
 #### (Optional) Performance configuration
-For optimal performance, you may use [PGTune](https://pgtune.leopard.in.ua), don't forget to restart postgresql after editing the configuration
+It is encouraged to check [Optimizing your PostgreSQL performance](../configuration/postgresql.md) document, for tips on PostgreSQL tuning.
 
 ```sh tab="Alpine"
 rc-service postgresql restart

installation/nginx-cache-purge.sh.example

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+# A simple shell script to delete a media from the Nginx cache.
+
+SCRIPTNAME=${0##*/}
+
+# NGINX cache directory
+CACHE_DIRECTORY="/tmp/pleroma-media-cache"
+
+## Return the files where the items are cached.
+## $1 - the filename, can be a pattern .
+## $2 - the cache directory.
+## $3 - (optional) the number of parallel processes to run for grep.
+get_cache_files() {
+    local max_parallel=${3-16}
+    find $2 -maxdepth 2 -type d | xargs -P $max_parallel -n 1 grep -E Rl "^KEY:.*$1" | sort -u
+}
+
+## Removes an item from the given cache zone.
+## $1 - the filename, can be a pattern .
+## $2 - the cache directory.
+purge_item() {
+  for f in $(get_cache_files $1 $2); do
+      echo "found file: $f"
+      [ -f $f ] || continue
+      echo "Deleting $f from $2."
+      rm $f
+  done
+} # purge_item
+
+purge() {
+  for url in "$@"
+  do
+    echo "$SCRIPTNAME delete \`$url\` from cache ($CACHE_DIRECTORY)"
+    purge_item $url $CACHE_DIRECTORY
+  done
+
+}
+
+purge $1

lib/mix/tasks/pleroma/database.ex

@@ -4,6 +4,7 @@
 
 defmodule Mix.Tasks.Pleroma.Database do
   alias Pleroma.Conversation
+  alias Pleroma.Maintenance
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
@@ -34,13 +35,7 @@ defmodule Mix.Tasks.Pleroma.Database do
     )
 
     if Keyword.get(options, :vacuum) do
-      Logger.info("Runnning VACUUM FULL")
+      Maintenance.vacuum("full")
-
-      Repo.query!(
-        "vacuum full;",
-        [],
-        timeout: :infinity
-      )
     end
   end
 
@@ -94,13 +89,7 @@ defmodule Mix.Tasks.Pleroma.Database do
     |> Repo.delete_all(timeout: :infinity)
 
     if Keyword.get(options, :vacuum) do
-      Logger.info("Runnning VACUUM FULL")
+      Maintenance.vacuum("full")
-
-      Repo.query!(
-        "vacuum full;",
-        [],
-        timeout: :infinity
-      )
     end
   end
 
@@ -135,4 +124,10 @@ defmodule Mix.Tasks.Pleroma.Database do
     end)
     |> Stream.run()
   end
+
+  def run(["vacuum", args]) do
+    start_pleroma()
+
+    Maintenance.vacuum(args)
+  end
 end

lib/mix/tasks/pleroma/emoji.ex

@@ -15,7 +15,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
     {options, [], []} = parse_global_opts(args)
 
     url_or_path = options[:manifest] || default_manifest()
-    manifest = fetch_manifest(url_or_path)
+    manifest = fetch_and_decode(url_or_path)
 
     Enum.each(manifest, fn {name, info} ->
       to_print = [
@@ -42,12 +42,12 @@ defmodule Mix.Tasks.Pleroma.Emoji do
 
     url_or_path = options[:manifest] || default_manifest()
 
-    manifest = fetch_manifest(url_or_path)
+    manifest = fetch_and_decode(url_or_path)
 
     for pack_name <- pack_names do
       if Map.has_key?(manifest, pack_name) do
         pack = manifest[pack_name]
-        src_url = pack["src"]
+        src = pack["src"]
 
         IO.puts(
           IO.ANSI.format([
@@ -57,11 +57,11 @@ defmodule Mix.Tasks.Pleroma.Emoji do
             :normal,
             " from ",
             :underline,
-            src_url
+            src
           ])
         )
 
-        binary_archive = Tesla.get!(client(), src_url).body
+        {:ok, binary_archive} = fetch(src)
         archive_sha = :crypto.hash(:sha256, binary_archive) |> Base.encode16()
 
         sha_status_text = ["SHA256 of ", :bright, pack_name, :normal, " source file is ", :bright]
@@ -74,8 +74,8 @@ defmodule Mix.Tasks.Pleroma.Emoji do
           raise "Bad SHA256 for #{pack_name}"
         end
 
-        # The url specified in files should be in the same directory
+        # The location specified in files should be in the same directory
-        files_url =
+        files_loc =
           url_or_path
           |> Path.dirname()
           |> Path.join(pack["files"])
@@ -88,11 +88,11 @@ defmodule Mix.Tasks.Pleroma.Emoji do
             :normal,
             " from ",
             :underline,
-            files_url
+            files_loc
           ])
         )
 
-        files = Tesla.get!(client(), files_url).body |> Jason.decode!()
+        files = fetch_and_decode(files_loc)
 
         IO.puts(IO.ANSI.format(["Unpacking ", :bright, pack_name]))
 
@@ -237,16 +237,20 @@ defmodule Mix.Tasks.Pleroma.Emoji do
     end
   end
 
-  defp fetch_manifest(from) do
+  defp fetch_and_decode(from) do
-    Jason.decode!(
+    with {:ok, json} <- fetch(from) do
-      if String.starts_with?(from, "http") do
+      Jason.decode!(json)
-        Tesla.get!(client(), from).body
+    end
-      else
-        File.read!(from)
-      end
-    )
   end
 
+  defp fetch("http" <> _ = from) do
+    with {:ok, %{body: body}} <- Tesla.get(client(), from) do
+      {:ok, body}
+    end
+  end
+
+  defp fetch(path), do: File.read(path)
+
   defp parse_global_opts(args) do
     OptionParser.parse(
       args,

lib/mix/tasks/pleroma/instance.ex

@@ -147,6 +147,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
           "What directory should media uploads go in (when using the local uploader)?",
           Pleroma.Config.get([Pleroma.Uploaders.Local, :uploads])
         )
+        |> Path.expand()
 
       static_dir =
         get_option(
@@ -155,6 +156,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
           "What directory should custom public files be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)?",
           Pleroma.Config.get([:instance, :static_dir])
         )
+        |> Path.expand()
 
       Config.put([:instance, :static_dir], static_dir)
 
@@ -204,7 +206,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
       shell_info("Writing the postgres script to #{psql_path}.")
       File.write(psql_path, result_psql)
 
-      write_robots_txt(indexable, template_dir)
+      write_robots_txt(static_dir, indexable, template_dir)
 
       shell_info(
         "\n All files successfully written! Refer to the installation instructions for your platform for next steps."
@@ -224,15 +226,13 @@ defmodule Mix.Tasks.Pleroma.Instance do
     end
   end
 
-  defp write_robots_txt(indexable, template_dir) do
+  defp write_robots_txt(static_dir, indexable, template_dir) do
     robots_txt =
       EEx.eval_file(
         template_dir <> "/robots_txt.eex",
         indexable: indexable
       )
 
-    static_dir = Pleroma.Config.get([:instance, :static_dir], "instance/static/")
-
     unless File.exists?(static_dir) do
       File.mkdir_p!(static_dir)
     end

lib/mix/tasks/pleroma/user.ex

@@ -144,28 +144,18 @@ defmodule Mix.Tasks.Pleroma.User do
     end
   end
 
-  def run(["unsubscribe", nickname]) do
+  def run(["deactivate", nickname]) do
     start_pleroma()
 
     with %User{} = user <- User.get_cached_by_nickname(nickname) do
       shell_info("Deactivating #{user.nickname}")
       User.deactivate(user)
-
-      user
-      |> User.get_friends()
-      |> Enum.each(fn friend ->
-        user = User.get_cached_by_id(user.id)
-
-        shell_info("Unsubscribing #{friend.nickname} from #{user.nickname}")
-        User.unfollow(user, friend)
-      end)
-
       :timer.sleep(500)
 
       user = User.get_cached_by_id(user.id)
 
-      if Enum.empty?(User.get_friends(user)) do
+      if Enum.empty?(Enum.filter(User.get_friends(user), & &1.local)) do
-        shell_info("Successfully unsubscribed all followers from #{user.nickname}")
+        shell_info("Successfully unsubscribed all local followers from #{user.nickname}")
       end
     else
       _ ->
@@ -173,7 +163,7 @@ defmodule Mix.Tasks.Pleroma.User do
     end
   end
 
-  def run(["unsubscribe_all_from_instance", instance]) do
+  def run(["deactivate_all_from_instance", instance]) do
     start_pleroma()
 
     Pleroma.User.Query.build(%{nickname: "@#{instance}"})
@@ -181,7 +171,7 @@ defmodule Mix.Tasks.Pleroma.User do
     |> Stream.each(fn users ->
       users
       |> Enum.each(fn user ->
-        run(["unsubscribe", user.nickname])
+        run(["deactivate", user.nickname])
       end)
     end)
     |> Stream.run()

lib/pleroma/activity/queries.ex

@@ -24,10 +24,7 @@ defmodule Pleroma.Activity.Queries do
 
   @spec by_actor(query, String.t()) :: query
   def by_actor(query \\ Activity, actor) do
-    from(
+    from(a in query, where: a.actor == ^actor)
-      activity in query,
-      where: fragment("(?)->>'actor' = ?", activity.data, ^actor)
-    )
   end
 
   @spec by_author(query, User.t()) :: query

lib/pleroma/bbs/handler.ex

@@ -92,10 +92,10 @@ defmodule Pleroma.BBS.Handler do
 
     params =
       %{}
-      |> Map.put("type", ["Create"])
+      |> Map.put(:type, ["Create"])
-      |> Map.put("blocking_user", user)
+      |> Map.put(:blocking_user, user)
-      |> Map.put("muting_user", user)
+      |> Map.put(:muting_user, user)
-      |> Map.put("user", user)
+      |> Map.put(:user, user)
 
     activities =
       [user.ap_id | Pleroma.User.following(user)]

lib/pleroma/constants.ex

@@ -24,6 +24,6 @@ defmodule Pleroma.Constants do
 
   const(static_only_files,
     do:
-      ~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc)
+      ~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc embed.js embed.css)
   )
 end

lib/pleroma/conversation.ex

@@ -63,7 +63,7 @@ defmodule Pleroma.Conversation do
          ap_id when is_binary(ap_id) and byte_size(ap_id) > 0 <- object.data["context"] do
       {:ok, conversation} = create_for_ap_id(ap_id)
 
-      users = User.get_users_from_set(activity.recipients, false)
+      users = User.get_users_from_set(activity.recipients, local_only: false)
 
       participations =
         Enum.map(users, fn user ->

lib/pleroma/conversation/participation.ex

@@ -163,8 +163,8 @@ defmodule Pleroma.Conversation.Participation do
     |> Enum.map(fn participation ->
       activity_id =
         ActivityPub.fetch_latest_activity_id_for_context(participation.conversation.ap_id, %{
-          "user" => user,
+          user: user,
-          "blocking_user" => user
+          blocking_user: user
         })
 
       %{

lib/pleroma/emails/new_users_digest_email.ex

@@ -14,8 +14,10 @@ defmodule Pleroma.Emails.NewUsersDigestEmail do
     styling = Pleroma.Config.get([Pleroma.Emails.UserEmail, :styling])
 
     logo_url =
-      Pleroma.Web.Endpoint.url() <>
+      Pleroma.Helpers.UriHelper.maybe_add_base(
-        Pleroma.Config.get([:frontend_configurations, :pleroma_fe, :logo])
+        Pleroma.Config.get([:frontend_configurations, :pleroma_fe, :logo]),
+        Pleroma.Web.Endpoint.url()
+      )
 
     new()
     |> to({to.name, to.email})

lib/pleroma/emoji/pack.ex

@@ -16,162 +16,78 @@ defmodule Pleroma.Emoji.Pack do
 
   alias Pleroma.Emoji
 
-  @spec emoji_path() :: Path.t()
-  def emoji_path do
-    static = Pleroma.Config.get!([:instance, :static_dir])
-    Path.join(static, "emoji")
-  end
-
   @spec create(String.t()) :: :ok | {:error, File.posix()} | {:error, :empty_values}
-  def create(name) when byte_size(name) > 0 do
+  def create(name) do
-    dir = Path.join(emoji_path(), name)
+    with :ok <- validate_not_empty([name]),
-
+         dir <- Path.join(emoji_path(), name),
-    with :ok <- File.mkdir(dir) do
+         :ok <- File.mkdir(dir) do
-      %__MODULE__{
+      %__MODULE__{pack_file: Path.join(dir, "pack.json")}
-        pack_file: Path.join(dir, "pack.json")
-      }
       |> save_pack()
     end
   end
 
-  def create(_), do: {:error, :empty_values}
+  @spec show(String.t()) :: {:ok, t()} | {:error, atom()}
-
+  def show(name) do
-  @spec show(String.t()) :: {:ok, t()} | {:loaded, nil} | {:error, :empty_values}
+    with :ok <- validate_not_empty([name]),
-  def show(name) when byte_size(name) > 0 do
+         {:ok, pack} <- load_pack(name) do
-    with {_, %__MODULE__{} = pack} <- {:loaded, load_pack(name)},
+      {:ok, validate_pack(pack)}
-         {_, pack} <- validate_pack(pack) do
-      {:ok, pack}
     end
   end
 
-  def show(_), do: {:error, :empty_values}
-
   @spec delete(String.t()) ::
           {:ok, [binary()]} | {:error, File.posix(), binary()} | {:error, :empty_values}
-  def delete(name) when byte_size(name) > 0 do
+  def delete(name) do
-    emoji_path()
+    with :ok <- validate_not_empty([name]) do
-    |> Path.join(name)
+      emoji_path()
-    |> File.rm_rf()
+      |> Path.join(name)
-  end
+      |> File.rm_rf()
-
-  def delete(_), do: {:error, :empty_values}
-
-  @spec add_file(String.t(), String.t(), Path.t(), Plug.Upload.t() | String.t()) ::
-          {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
-  def add_file(name, shortcode, filename, file)
-      when byte_size(name) > 0 and byte_size(shortcode) > 0 and byte_size(filename) > 0 do
-    with {_, nil} <- {:exists, Emoji.get(shortcode)},
-         {_, %__MODULE__{} = pack} <- {:loaded, load_pack(name)} do
-      file_path = Path.join(pack.path, filename)
-
-      create_subdirs(file_path)
-
-      case file do
-        %Plug.Upload{path: upload_path} ->
-          # Copy the uploaded file from the temporary directory
-          File.copy!(upload_path, file_path)
-
-        url when is_binary(url) ->
-          # Download and write the file
-          file_contents = Tesla.get!(url).body
-          File.write!(file_path, file_contents)
-      end
-
-      files = Map.put(pack.files, shortcode, filename)
-
-      updated_pack = %{pack | files: files}
-
-      case save_pack(updated_pack) do
-        :ok ->
-          Emoji.reload()
-          {:ok, updated_pack}
-
-        e ->
-          e
-      end
     end
   end
 
-  def add_file(_, _, _, _), do: {:error, :empty_values}
+  @spec add_file(String.t(), String.t(), Path.t(), Plug.Upload.t() | String.t()) ::
-
+          {:ok, t()} | {:error, File.posix() | atom()}
-  defp create_subdirs(file_path) do
+  def add_file(name, shortcode, filename, file) do
-    if String.contains?(file_path, "/") do
+    with :ok <- validate_not_empty([name, shortcode, filename]),
-      file_path
+         :ok <- validate_emoji_not_exists(shortcode),
-      |> Path.dirname()
+         {:ok, pack} <- load_pack(name),
-      |> File.mkdir_p!()
+         :ok <- save_file(file, pack, filename),
+         {:ok, updated_pack} <- pack |> put_emoji(shortcode, filename) |> save_pack() do
+      Emoji.reload()
+      {:ok, updated_pack}
     end
   end
 
   @spec delete_file(String.t(), String.t()) ::
-          {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
+          {:ok, t()} | {:error, File.posix() | atom()}
-  def delete_file(name, shortcode) when byte_size(name) > 0 and byte_size(shortcode) > 0 do
+  def delete_file(name, shortcode) do
-    with {_, %__MODULE__{} = pack} <- {:loaded, load_pack(name)},
+    with :ok <- validate_not_empty([name, shortcode]),
-         {_, {filename, files}} when not is_nil(filename) <-
+         {:ok, pack} <- load_pack(name),
-           {:exists, Map.pop(pack.files, shortcode)},
+         :ok <- remove_file(pack, shortcode),
-         emoji <- Path.join(pack.path, filename),
+         {:ok, updated_pack} <- pack |> delete_emoji(shortcode) |> save_pack() do
-         {_, true} <- {:exists, File.exists?(emoji)} do
+      Emoji.reload()
-      emoji_dir = Path.dirname(emoji)
+      {:ok, updated_pack}
-
-      File.rm!(emoji)
-
-      if String.contains?(filename, "/") and File.ls!(emoji_dir) == [] do
-        File.rmdir!(emoji_dir)
-      end
-
-      updated_pack = %{pack | files: files}
-
-      case save_pack(updated_pack) do
-        :ok ->
-          Emoji.reload()
-          {:ok, updated_pack}
-
-        e ->
-          e
-      end
     end
   end
 
-  def delete_file(_, _), do: {:error, :empty_values}
-
   @spec update_file(String.t(), String.t(), String.t(), String.t(), boolean()) ::
-          {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
+          {:ok, t()} | {:error, File.posix() | atom()}
-  def update_file(name, shortcode, new_shortcode, new_filename, force)
+  def update_file(name, shortcode, new_shortcode, new_filename, force) do
-      when byte_size(name) > 0 and byte_size(shortcode) > 0 and byte_size(new_shortcode) > 0 and
+    with :ok <- validate_not_empty([name, shortcode, new_shortcode, new_filename]),
-             byte_size(new_filename) > 0 do
+         {:ok, pack} <- load_pack(name),
-    with {_, %__MODULE__{} = pack} <- {:loaded, load_pack(name)},
+         {:ok, filename} <- get_filename(pack, shortcode),
-         {_, {filename, files}} when not is_nil(filename) <-
+         :ok <- validate_emoji_not_exists(new_shortcode, force),
-           {:exists, Map.pop(pack.files, shortcode)},
+         :ok <- rename_file(pack, filename, new_filename),
-         {_, true} <- {:not_used, force or is_nil(Emoji.get(new_shortcode))} do
+         {:ok, updated_pack} <-
-      old_path = Path.join(pack.path, filename)
+           pack
-      old_dir = Path.dirname(old_path)
+           |> delete_emoji(shortcode)
-      new_path = Path.join(pack.path, new_filename)
+           |> put_emoji(new_shortcode, new_filename)
-
+           |> save_pack() do
-      create_subdirs(new_path)
+      Emoji.reload()
-
+      {:ok, updated_pack}
-      :ok = File.rename(old_path, new_path)
-
-      if String.contains?(filename, "/") and File.ls!(old_dir) == [] do
-        File.rmdir!(old_dir)
-      end
-
-      files = Map.put(files, new_shortcode, new_filename)
-
-      updated_pack = %{pack | files: files}
-
-      case save_pack(updated_pack) do
-        :ok ->
-          Emoji.reload()
-          {:ok, updated_pack}
-
-        e ->
-          e
-      end
     end
   end
 
-  def update_file(_, _, _, _, _), do: {:error, :empty_values}
+  @spec import_from_filesystem() :: {:ok, [String.t()]} | {:error, File.posix() | atom()}
-
-  @spec import_from_filesystem() :: {:ok, [String.t()]} | {:error, atom()}
   def import_from_filesystem do
     emoji_path = emoji_path()
 
@@ -184,7 +100,7 @@ defmodule Pleroma.Emoji.Pack do
           File.dir?(path) and File.exists?(Path.join(path, "pack.json"))
         end)
         |> Enum.map(&write_pack_contents/1)
-        |> Enum.filter(& &1)
+        |> Enum.reject(&is_nil/1)
 
       {:ok, names}
     else
@@ -193,6 +109,117 @@ defmodule Pleroma.Emoji.Pack do
     end
   end
 
+  @spec list_remote(String.t()) :: {:ok, map()} | {:error, atom()}
+  def list_remote(url) do
+    uri = url |> String.trim() |> URI.parse()
+
+    with :ok <- validate_shareable_packs_available(uri) do
+      uri
+      |> URI.merge("/api/pleroma/emoji/packs")
+      |> http_get()
+    end
+  end
+
+  @spec list_local() :: {:ok, map()}
+  def list_local do
+    with {:ok, results} <- list_packs_dir() do
+      packs =
+        results
+        |> Enum.map(fn name ->
+          case load_pack(name) do
+            {:ok, pack} -> pack
+            _ -> nil
+          end
+        end)
+        |> Enum.reject(&is_nil/1)
+        |> Map.new(fn pack -> {pack.name, validate_pack(pack)} end)
+
+      {:ok, packs}
+    end
+  end
+
+  @spec get_archive(String.t()) :: {:ok, binary()} | {:error, atom()}
+  def get_archive(name) do
+    with {:ok, pack} <- load_pack(name),
+         :ok <- validate_downloadable(pack) do
+      {:ok, fetch_archive(pack)}
+    end
+  end
+
+  @spec download(String.t(), String.t(), String.t()) :: :ok | {:error, atom()}
+  def download(name, url, as) do
+    uri = url |> String.trim() |> URI.parse()
+
+    with :ok <- validate_shareable_packs_available(uri),
+         {:ok, remote_pack} <- uri |> URI.merge("/api/pleroma/emoji/packs/#{name}") |> http_get(),
+         {:ok, %{sha: sha, url: url} = pack_info} <- fetch_pack_info(remote_pack, uri, name),
+         {:ok, archive} <- download_archive(url, sha),
+         pack <- copy_as(remote_pack, as || name),
+         {:ok, _} = unzip(archive, pack_info, remote_pack, pack) do
+      # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256
+      # in it to depend on itself
+      if pack_info[:fallback] do
+        save_pack(pack)
+      else
+        {:ok, pack}
+      end
+    end
+  end
+
+  @spec save_metadata(map(), t()) :: {:ok, t()} | {:error, File.posix()}
+  def save_metadata(metadata, %__MODULE__{} = pack) do
+    pack
+    |> Map.put(:pack, metadata)
+    |> save_pack()
+  end
+
+  @spec update_metadata(String.t(), map()) :: {:ok, t()} | {:error, File.posix()}
+  def update_metadata(name, data) do
+    with {:ok, pack} <- load_pack(name) do
+      if fallback_sha_changed?(pack, data) do
+        update_sha_and_save_metadata(pack, data)
+      else
+        save_metadata(data, pack)
+      end
+    end
+  end
+
+  @spec load_pack(String.t()) :: {:ok, t()} | {:error, :not_found}
+  def load_pack(name) do
+    pack_file = Path.join([emoji_path(), name, "pack.json"])
+
+    if File.exists?(pack_file) do
+      pack =
+        pack_file
+        |> File.read!()
+        |> from_json()
+        |> Map.put(:pack_file, pack_file)
+        |> Map.put(:path, Path.dirname(pack_file))
+        |> Map.put(:name, name)
+
+      {:ok, pack}
+    else
+      {:error, :not_found}
+    end
+  end
+
+  @spec emoji_path() :: Path.t()
+  defp emoji_path do
+    [:instance, :static_dir]
+    |> Pleroma.Config.get!()
+    |> Path.join("emoji")
+  end
+
+  defp validate_emoji_not_exists(shortcode, force \\ false)
+  defp validate_emoji_not_exists(_shortcode, true), do: :ok
+
+  defp validate_emoji_not_exists(shortcode, _) do
+    case Emoji.get(shortcode) do
+      nil -> :ok
+      _ -> {:error, :already_exists}
+    end
+  end
+
   defp write_pack_contents(path) do
     pack = %__MODULE__{
       files: files_from_path(path),
@@ -201,7 +228,7 @@ defmodule Pleroma.Emoji.Pack do
     }
 
     case save_pack(pack) do
-      :ok -> Path.basename(path)
+      {:ok, _pack} -> Path.basename(path)
       _ -> nil
     end
   end
@@ -216,7 +243,8 @@ defmodule Pleroma.Emoji.Pack do
       # FIXME: Copy-pasted from Pleroma.Emoji/load_from_file_stream/2
 
       # Create a map of shortcodes to filenames from emoji.txt
-      File.read!(txt_path)
+      txt_path
+      |> File.read!()
       |> String.split("\n")
       |> Enum.map(&String.trim/1)
       |> Enum.map(fn line ->
@@ -226,21 +254,18 @@ defmodule Pleroma.Emoji.Pack do
           [name, file | _] ->
             file_dir_name = Path.dirname(file)
 
-            file =
+            if String.ends_with?(path, file_dir_name) do
-              if String.ends_with?(path, file_dir_name) do
+              {name, Path.basename(file)}
-                Path.basename(file)
+            else
-              else
+              {name, file}
-                file
+            end
-              end
-
-            {name, file}
 
           _ ->
             nil
         end
       end)
-      |> Enum.filter(& &1)
+      |> Enum.reject(&is_nil/1)
-      |> Enum.into(%{})
+      |> Map.new()
     else
       # If there's no emoji.txt, assume all files
       # that are of certain extensions from the config are emojis and import them all
@@ -249,60 +274,20 @@ defmodule Pleroma.Emoji.Pack do
     end
   end
 
-  @spec list_remote(String.t()) :: {:ok, map()}
-  def list_remote(url) do
-    uri =
-      url
-      |> String.trim()
-      |> URI.parse()
-
-    with {_, true} <- {:shareable, shareable_packs_available?(uri)} do
-      packs =
-        uri
-        |> URI.merge("/api/pleroma/emoji/packs")
-        |> to_string()
-        |> Tesla.get!()
-        |> Map.get(:body)
-        |> Jason.decode!()
-
-      {:ok, packs}
-    end
-  end
-
-  @spec list_local() :: {:ok, map()}
-  def list_local do
-    emoji_path = emoji_path()
-
-    # Create the directory first if it does not exist. This is probably the first request made
-    # with the API so it should be sufficient
-    with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(emoji_path)},
-         {:ls, {:ok, results}} <- {:ls, File.ls(emoji_path)} do
-      packs =
-        results
-        |> Enum.map(&load_pack/1)
-        |> Enum.filter(& &1)
-        |> Enum.map(&validate_pack/1)
-        |> Map.new()
-
-      {:ok, packs}
-    end
-  end
-
   defp validate_pack(pack) do
-    if downloadable?(pack) do
+    info =
-      archive = fetch_archive(pack)
+      if downloadable?(pack) do
-      archive_sha = :crypto.hash(:sha256, archive) |> Base.encode16()
+        archive = fetch_archive(pack)
+        archive_sha = :crypto.hash(:sha256, archive) |> Base.encode16()
 
-      info =
         pack.pack
         |> Map.put("can-download", true)
         |> Map.put("download-sha256", archive_sha)
+      else
+        Map.put(pack.pack, "can-download", false)
+      end
 
-      {pack.name, Map.put(pack, :pack, info)}
+    Map.put(pack, :pack, info)
-    else
-      info = Map.put(pack.pack, "can-download", false)
-      {pack.name, Map.put(pack, :pack, info)}
-    end
   end
 
   defp downloadable?(pack) do
@@ -315,26 +300,6 @@ defmodule Pleroma.Emoji.Pack do
       end)
   end
 
-  @spec get_archive(String.t()) :: {:ok, binary()}
-  def get_archive(name) do
-    with {_, %__MODULE__{} = pack} <- {:exists?, load_pack(name)},
-         {_, true} <- {:can_download?, downloadable?(pack)} do
-      {:ok, fetch_archive(pack)}
-    end
-  end
-
-  defp fetch_archive(pack) do
-    hash = :crypto.hash(:md5, File.read!(pack.pack_file))
-
-    case Cachex.get!(:emoji_packs_cache, pack.name) do
-      %{hash: ^hash, pack_data: archive} ->
-        archive
-
-      _ ->
-        create_archive_and_cache(pack, hash)
-    end
-  end
-
   defp create_archive_and_cache(pack, hash) do
     files = ['pack.json' | Enum.map(pack.files, fn {_, file} -> to_charlist(file) end)]
 
@@ -356,152 +321,221 @@ defmodule Pleroma.Emoji.Pack do
     result
   end
 
-  @spec download(String.t(), String.t(), String.t()) :: :ok
+  defp save_pack(pack) do
-  def download(name, url, as) do
+    with {:ok, json} <- Jason.encode(pack, pretty: true),
-    uri =
+         :ok <- File.write(pack.pack_file, json) do
-      url
-      |> String.trim()
-      |> URI.parse()
-
-    with {_, true} <- {:shareable, shareable_packs_available?(uri)} do
-      remote_pack =
-        uri
-        |> URI.merge("/api/pleroma/emoji/packs/#{name}")
-        |> to_string()
-        |> Tesla.get!()
-        |> Map.get(:body)
-        |> Jason.decode!()
-
-      result =
-        case remote_pack["pack"] do
-          %{"share-files" => true, "can-download" => true, "download-sha256" => sha} ->
-            {:ok,
-             %{
-               sha: sha,
-               url: URI.merge(uri, "/api/pleroma/emoji/packs/#{name}/archive") |> to_string()
-             }}
-
-          %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) ->
-            {:ok,
-             %{
-               sha: sha,
-               url: src,
-               fallback: true
-             }}
-
-          _ ->
-            {:error,
-             "The pack was not set as shared and there is no fallback src to download from"}
-        end
-
-      with {:ok, %{sha: sha, url: url} = pinfo} <- result,
-           %{body: archive} <- Tesla.get!(url),
-           {_, true} <- {:checksum, Base.decode16!(sha) == :crypto.hash(:sha256, archive)} do
-        local_name = as || name
-
-        path = Path.join(emoji_path(), local_name)
-
-        pack = %__MODULE__{
-          name: local_name,
-          path: path,
-          files: remote_pack["files"],
-          pack_file: Path.join(path, "pack.json")
-        }
-
-        File.mkdir_p!(pack.path)
-
-        files = Enum.map(remote_pack["files"], fn {_, path} -> to_charlist(path) end)
-        # Fallback cannot contain a pack.json file
-        files = if pinfo[:fallback], do: files, else: ['pack.json' | files]
-
-        {:ok, _} = :zip.unzip(archive, cwd: to_charlist(pack.path), file_list: files)
-
-        # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256
-        # in it to depend on itself
-        if pinfo[:fallback] do
-          save_pack(pack)
-        end
-
-        :ok
-      end
-    end
-  end
-
-  defp save_pack(pack), do: File.write(pack.pack_file, Jason.encode!(pack, pretty: true))
-
-  @spec save_metadata(map(), t()) :: {:ok, t()} | {:error, File.posix()}
-  def save_metadata(metadata, %__MODULE__{} = pack) do
-    pack = Map.put(pack, :pack, metadata)
-
-    with :ok <- save_pack(pack) do
       {:ok, pack}
     end
   end
 
-  @spec update_metadata(String.t(), map()) :: {:ok, t()} | {:error, File.posix()}
-  def update_metadata(name, data) do
-    pack = load_pack(name)
-
-    fb_sha_changed? =
-      not is_nil(data["fallback-src"]) and data["fallback-src"] != pack.pack["fallback-src"]
-
-    with {_, true} <- {:update?, fb_sha_changed?},
-         {:ok, %{body: zip}} <- Tesla.get(data["fallback-src"]),
-         {:ok, f_list} <- :zip.unzip(zip, [:memory]),
-         {_, true} <- {:has_all_files?, has_all_files?(pack.files, f_list)} do
-      fallback_sha = :crypto.hash(:sha256, zip) |> Base.encode16()
-
-      data
-      |> Map.put("fallback-src-sha256", fallback_sha)
-      |> save_metadata(pack)
-    else
-      {:update?, _} -> save_metadata(data, pack)
-      e -> e
-    end
-  end
-
-  # Check if all files from the pack.json are in the archive
-  defp has_all_files?(files, f_list) do
-    Enum.all?(files, fn {_, from_manifest} ->
-      List.keyfind(f_list, to_charlist(from_manifest), 0)
-    end)
-  end
-
-  @spec load_pack(String.t()) :: t() | nil
-  def load_pack(name) do
-    pack_file = Path.join([emoji_path(), name, "pack.json"])
-
-    if File.exists?(pack_file) do
-      pack_file
-      |> File.read!()
-      |> from_json()
-      |> Map.put(:pack_file, pack_file)
-      |> Map.put(:path, Path.dirname(pack_file))
-      |> Map.put(:name, name)
-    end
-  end
-
   defp from_json(json) do
     map = Jason.decode!(json)
 
     struct(__MODULE__, %{files: map["files"], pack: map["pack"]})
   end
 
-  defp shareable_packs_available?(uri) do
+  defp validate_shareable_packs_available(uri) do
-    uri
+    with {:ok, %{"links" => links}} <- uri |> URI.merge("/.well-known/nodeinfo") |> http_get(),
-    |> URI.merge("/.well-known/nodeinfo")
+         # Get the actual nodeinfo address and fetch it
-    |> to_string()
+         {:ok, %{"metadata" => %{"features" => features}}} <-
-    |> Tesla.get!()
+           links |> List.last() |> Map.get("href") |> http_get() do
-    |> Map.get(:body)
+      if Enum.member?(features, "shareable_emoji_packs") do
-    |> Jason.decode!()
+        :ok
-    |> Map.get("links")
+      else
-    |> List.last()
+        {:error, :not_shareable}
-    |> Map.get("href")
+      end
-    # Get the actual nodeinfo address and fetch it
+    end
-    |> Tesla.get!()
+  end
-    |> Map.get(:body)
+
-    |> Jason.decode!()
+  defp validate_not_empty(list) do
-    |> get_in(["metadata", "features"])
+    if Enum.all?(list, fn i -> is_binary(i) and i != "" end) do
-    |> Enum.member?("shareable_emoji_packs")
+      :ok
+    else
+      {:error, :empty_values}
+    end
+  end
+
+  defp save_file(file, pack, filename) do
+    file_path = Path.join(pack.path, filename)
+    create_subdirs(file_path)
+
+    case file do
+      %Plug.Upload{path: upload_path} ->
+        # Copy the uploaded file from the temporary directory
+        with {:ok, _} <- File.copy(upload_path, file_path), do: :ok
+
+      url when is_binary(url) ->
+        # Download and write the file
+        file_contents = Tesla.get!(url).body
+        File.write(file_path, file_contents)
+    end
+  end
+
+  defp put_emoji(pack, shortcode, filename) do
+    files = Map.put(pack.files, shortcode, filename)
+    %{pack | files: files}
+  end
+
+  defp delete_emoji(pack, shortcode) do
+    files = Map.delete(pack.files, shortcode)
+    %{pack | files: files}
+  end
+
+  defp rename_file(pack, filename, new_filename) do
+    old_path = Path.join(pack.path, filename)
+    new_path = Path.join(pack.path, new_filename)
+    create_subdirs(new_path)
+
+    with :ok <- File.rename(old_path, new_path) do
+      remove_dir_if_empty(old_path, filename)
+    end
+  end
+
+  defp create_subdirs(file_path) do
+    if String.contains?(file_path, "/") do
+      file_path
+      |> Path.dirname()
+      |> File.mkdir_p!()
+    end
+  end
+
+  defp remove_file(pack, shortcode) do
+    with {:ok, filename} <- get_filename(pack, shortcode),
+         emoji <- Path.join(pack.path, filename),
+         :ok <- File.rm(emoji) do
+      remove_dir_if_empty(emoji, filename)
+    end
+  end
+
+  defp remove_dir_if_empty(emoji, filename) do
+    dir = Path.dirname(emoji)
+
+    if String.contains?(filename, "/") and File.ls!(dir) == [] do
+      File.rmdir!(dir)
+    else
+      :ok
+    end
+  end
+
+  defp get_filename(pack, shortcode) do
+    with %{^shortcode => filename} when is_binary(filename) <- pack.files,
+         true <- pack.path |> Path.join(filename) |> File.exists?() do
+      {:ok, filename}
+    else
+      _ -> {:error, :doesnt_exist}
+    end
+  end
+
+  defp http_get(%URI{} = url), do: url |> to_string() |> http_get()
+
+  defp http_get(url) do
+    with {:ok, %{body: body}} <- url |> Pleroma.HTTP.get() do
+      Jason.decode(body)
+    end
+  end
+
+  defp list_packs_dir do
+    emoji_path = emoji_path()
+    # Create the directory first if it does not exist. This is probably the first request made
+    # with the API so it should be sufficient
+    with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(emoji_path)},
+         {:ls, {:ok, results}} <- {:ls, File.ls(emoji_path)} do
+      {:ok, results}
+    else
+      {:create_dir, {:error, e}} -> {:error, :create_dir, e}
+      {:ls, {:error, e}} -> {:error, :ls, e}
+    end
+  end
+
+  defp validate_downloadable(pack) do
+    if downloadable?(pack), do: :ok, else: {:error, :cant_download}
+  end
+
+  defp copy_as(remote_pack, local_name) do
+    path = Path.join(emoji_path(), local_name)
+
+    %__MODULE__{
+      name: local_name,
+      path: path,
+      files: remote_pack["files"],
+      pack_file: Path.join(path, "pack.json")
+    }
+  end
+
+  defp unzip(archive, pack_info, remote_pack, local_pack) do
+    with :ok <- File.mkdir_p!(local_pack.path) do
+      files = Enum.map(remote_pack["files"], fn {_, path} -> to_charlist(path) end)
+      # Fallback cannot contain a pack.json file
+      files = if pack_info[:fallback], do: files, else: ['pack.json' | files]
+
+      :zip.unzip(archive, cwd: to_charlist(local_pack.path), file_list: files)
+    end
+  end
+
+  defp fetch_pack_info(remote_pack, uri, name) do
+    case remote_pack["pack"] do
+      %{"share-files" => true, "can-download" => true, "download-sha256" => sha} ->
+        {:ok,
+         %{
+           sha: sha,
+           url: URI.merge(uri, "/api/pleroma/emoji/packs/#{name}/archive") |> to_string()
+         }}
+
+      %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) ->
+        {:ok,
+         %{
+           sha: sha,
+           url: src,
+           fallback: true
+         }}
+
+      _ ->
+        {:error, "The pack was not set as shared and there is no fallback src to download from"}
+    end
+  end
+
+  defp download_archive(url, sha) do
+    with {:ok, %{body: archive}} <- Tesla.get(url) do
+      if Base.decode16!(sha) == :crypto.hash(:sha256, archive) do
+        {:ok, archive}
+      else
+        {:error, :invalid_checksum}
+      end
+    end
+  end
+
+  defp fetch_archive(pack) do
+    hash = :crypto.hash(:md5, File.read!(pack.pack_file))
+
+    case Cachex.get!(:emoji_packs_cache, pack.name) do
+      %{hash: ^hash, pack_data: archive} -> archive
+      _ -> create_archive_and_cache(pack, hash)
+    end
+  end
+
+  defp fallback_sha_changed?(pack, data) do
+    is_binary(data[:"fallback-src"]) and data[:"fallback-src"] != pack.pack["fallback-src"]
+  end
+
+  defp update_sha_and_save_metadata(pack, data) do
+    with {:ok, %{body: zip}} <- Tesla.get(data[:"fallback-src"]),
+         :ok <- validate_has_all_files(pack, zip) do
+      fallback_sha = :sha256 |> :crypto.hash(zip) |> Base.encode16()
+
+      data
+      |> Map.put("fallback-src-sha256", fallback_sha)
+      |> save_metadata(pack)
+    end
+  end
+
+  defp validate_has_all_files(pack, zip) do
+    with {:ok, f_list} <- :zip.unzip(zip, [:memory]) do
+      # Check if all files from the pack.json are in the archive
+      pack.files
+      |> Enum.all?(fn {_, from_manifest} ->
+        List.keyfind(f_list, to_charlist(from_manifest), 0)
+      end)
+      |> if(do: :ok, else: {:error, :incomplete})
+    end
   end
 end

lib/pleroma/following_relationship.ex

@@ -141,6 +141,12 @@ defmodule Pleroma.FollowingRelationship do
     |> where([r], r.state == ^:follow_accept)
   end
 
+  def outgoing_pending_follow_requests_query(%User{} = follower) do
+    __MODULE__
+    |> where([r], r.follower_id == ^follower.id)
+    |> where([r], r.state == ^:follow_pending)
+  end
+
   def following(%User{} = user) do
     following =
       following_query(user)

lib/pleroma/helpers/uri_helper.ex

@@ -17,11 +17,6 @@ defmodule Pleroma.Helpers.UriHelper do
     |> URI.to_string()
   end
 
-  def append_param_if_present(%{} = params, param_name, param_value) do
+  def maybe_add_base("/" <> uri, base), do: Path.join([base, uri])
-    if param_value do
+  def maybe_add_base(uri, _base), do: uri
-      Map.put(params, param_name, param_value)
-    else
-      params
-    end
-  end
 end

lib/pleroma/http/adapter_helper/hackney.ex

@@ -22,22 +22,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
     |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
   end
 
-  defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts
+  defp add_scheme_opts(opts, _), do: opts
-
-  defp add_scheme_opts(opts, %URI{scheme: "https", host: host}) do
-    ssl_opts = [
-      ssl_options: [
-        # Workaround for remote server certificate chain issues
-        partial_chain: &:hackney_connect.partial_chain/1,
-
-        # We don't support TLS v1.3 yet
-        versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"],
-        server_name_indication: to_charlist(host)
-      ]
-    ]
-
-    Keyword.merge(opts, ssl_opts)
-  end
 
   def after_request(_), do: :ok
 end

lib/pleroma/maintenance.ex

@@ -0,0 +1,37 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Maintenance do
+  alias Pleroma.Repo
+  require Logger
+
+  def vacuum(args) do
+    case args do
+      "analyze" ->
+        Logger.info("Runnning VACUUM ANALYZE.")
+
+        Repo.query!(
+          "vacuum analyze;",
+          [],
+          timeout: :infinity
+        )
+
+      "full" ->
+        Logger.info("Runnning VACUUM FULL.")
+
+        Logger.warn(
+          "Re-packing your entire database may take a while and will consume extra disk space during the process."
+        )
+
+        Repo.query!(
+          "vacuum full;",
+          [],
+          timeout: :infinity
+        )
+
+      _ ->
+        Logger.error("Error: invalid vacuum argument.")
+    end
+  end
+end

lib/pleroma/maps.ex

@@ -0,0 +1,15 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Maps do
+  def put_if_present(map, key, value, value_function \\ &{:ok, &1}) when is_map(map) do
+    with false <- is_nil(key),
+         false <- is_nil(value),
+         {:ok, new_value} <- value_function.(value) do
+      Map.put(map, key, new_value)
+    else
+      _ -> map
+    end
+  end
+end

lib/pleroma/mfa.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.MFA do

lib/pleroma/mfa/backup_codes.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.MFA.BackupCodes do

lib/pleroma/mfa/changeset.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.MFA.Changeset do

lib/pleroma/mfa/settings.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.MFA.Settings do

lib/pleroma/mfa/token.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.MFA.Token do

lib/pleroma/mfa/totp.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.MFA.TOTP do

lib/pleroma/notification.ex

@@ -92,8 +92,9 @@ defmodule Pleroma.Notification do
     |> join(:left, [n, a], object in Object,
       on:
         fragment(
-          "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)",
+          "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
           object.data,
+          a.data,
           a.data
         )
     )
@@ -224,18 +225,8 @@ defmodule Pleroma.Notification do
       |> Marker.multi_set_last_read_id(user, "notifications")
       |> Repo.transaction()
 
-    Notification
+    for_user_query(user)
     |> where([n], n.id in ^notification_ids)
-    |> join(:inner, [n], activity in assoc(n, :activity))
-    |> join(:left, [n, a], object in Object,
-      on:
-        fragment(
-          "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)",
-          object.data,
-          a.data
-        )
-    )
-    |> preload([n, a, o], activity: {a, object: o})
     |> Repo.all()
   end
 
@@ -370,7 +361,8 @@ defmodule Pleroma.Notification do
       when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact"] do
     potential_receiver_ap_ids = get_potential_receiver_ap_ids(activity)
 
-    potential_receivers = User.get_users_from_set(potential_receiver_ap_ids, local_only)
+    potential_receivers =
+      User.get_users_from_set(potential_receiver_ap_ids, local_only: local_only)
 
     notification_enabled_ap_ids =
       potential_receiver_ap_ids

lib/pleroma/object.ex

@@ -9,11 +9,13 @@ defmodule Pleroma.Object do
   import Ecto.Changeset
 
   alias Pleroma.Activity
+  alias Pleroma.Config
   alias Pleroma.Object
   alias Pleroma.Object.Fetcher
   alias Pleroma.ObjectTombstone
   alias Pleroma.Repo
   alias Pleroma.User
+  alias Pleroma.Workers.AttachmentsCleanupWorker
 
   require Logger
 
@@ -138,12 +140,17 @@ defmodule Pleroma.Object do
 
   def normalize(_, _, _), do: nil
 
-  # Owned objects can only be mutated by their owner
+  # Owned objects can only be accessed by their owner
-  def authorize_mutation(%Object{data: %{"actor" => actor}}, %User{ap_id: ap_id}),
+  def authorize_access(%Object{data: %{"actor" => actor}}, %User{ap_id: ap_id}) do
-    do: actor == ap_id
+    if actor == ap_id do
+      :ok
+    else
+      {:error, :forbidden}
+    end
+  end
 
-  # Legacy objects can be mutated by anybody
+  # Legacy objects can be accessed by anybody
-  def authorize_mutation(%Object{}, %User{}), do: true
+  def authorize_access(%Object{}, %User{}), do: :ok
 
   @spec get_cached_by_ap_id(String.t()) :: Object.t() | nil
   def get_cached_by_ap_id(ap_id) do
@@ -183,27 +190,37 @@ defmodule Pleroma.Object do
   def delete(%Object{data: %{"id" => id}} = object) do
     with {:ok, _obj} = swap_object_with_tombstone(object),
          deleted_activity = Activity.delete_all_by_object_ap_id(id),
-         {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
+         {:ok, _} <- invalid_object_cache(object) do
-         {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
+      cleanup_attachments(
-      with true <- Pleroma.Config.get([:instance, :cleanup_attachments]) do
+        Config.get([:instance, :cleanup_attachments]),
-        {:ok, _} =
+        %{"object" => object}
-          Pleroma.Workers.AttachmentsCleanupWorker.enqueue("cleanup_attachments", %{
+      )
-            "object" => object
-          })
-      end
 
       {:ok, object, deleted_activity}
     end
   end
 
-  def prune(%Object{data: %{"id" => id}} = object) do
+  @spec cleanup_attachments(boolean(), %{required(:object) => map()}) ::
+          {:ok, Oban.Job.t() | nil}
+  def cleanup_attachments(true, %{"object" => _} = params) do
+    AttachmentsCleanupWorker.enqueue("cleanup_attachments", params)
+  end
+
+  def cleanup_attachments(_, _), do: {:ok, nil}
+
+  def prune(%Object{data: %{"id" => _id}} = object) do
     with {:ok, object} <- Repo.delete(object),
-         {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
+         {:ok, _} <- invalid_object_cache(object) do
-         {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
       {:ok, object}
     end
   end
 
+  def invalid_object_cache(%Object{data: %{"id" => id}}) do
+    with {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
+      Cachex.del(:web_resp_cache, URI.parse(id).path)
+    end
+  end
+
   def set_cache(%Object{data: %{"id" => ap_id}} = object) do
     Cachex.put(:object_cache, "object:#{ap_id}", object)
     {:ok, object}

lib/pleroma/pagination.ex

@@ -23,12 +23,12 @@ defmodule Pleroma.Pagination do
   @spec fetch_paginated(Ecto.Query.t(), map(), type(), atom() | nil) :: [Ecto.Schema.t()]
   def fetch_paginated(query, params, type \\ :keyset, table_binding \\ nil)
 
-  def fetch_paginated(query, %{"total" => true} = params, :keyset, table_binding) do
+  def fetch_paginated(query, %{total: true} = params, :keyset, table_binding) do
     total = Repo.aggregate(query, :count, :id)
 
     %{
       total: total,
-      items: fetch_paginated(query, Map.drop(params, ["total"]), :keyset, table_binding)
+      items: fetch_paginated(query, Map.drop(params, [:total]), :keyset, table_binding)
     }
   end
 
@@ -41,7 +41,7 @@ defmodule Pleroma.Pagination do
     |> enforce_order(options)
   end
 
-  def fetch_paginated(query, %{"total" => true} = params, :offset, table_binding) do
+  def fetch_paginated(query, %{total: true} = params, :offset, table_binding) do
     total =
       query
       |> Ecto.Query.exclude(:left_join)
@@ -49,7 +49,7 @@ defmodule Pleroma.Pagination do
 
     %{
       total: total,
-      items: fetch_paginated(query, Map.drop(params, ["total"]), :offset, table_binding)
+      items: fetch_paginated(query, Map.drop(params, [:total]), :offset, table_binding)
     }
   end
 
@@ -90,12 +90,6 @@ defmodule Pleroma.Pagination do
       skip_order: :boolean
     }
 
-    params =
-      Enum.reduce(params, %{}, fn
-        {key, _value}, acc when is_atom(key) -> Map.drop(acc, [key])
-        {key, value}, acc -> Map.put(acc, key, value)
-      end)
-
     changeset = cast({%{}, param_types}, params, Map.keys(param_types))
     changeset.changes
   end

lib/pleroma/plugs/http_security_plug.ex

@@ -31,7 +31,7 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do
       {"x-content-type-options", "nosniff"},
       {"referrer-policy", referrer_policy},
       {"x-download-options", "noopen"},
-      {"content-security-policy", csp_string() <> ";"}
+      {"content-security-policy", csp_string()}
     ]
 
     if report_uri do
@@ -43,23 +43,46 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do
         ]
       }
 
-      headers ++ [{"reply-to", Jason.encode!(report_group)}]
+      [{"reply-to", Jason.encode!(report_group)} | headers]
     else
       headers
     end
   end
 
+  static_csp_rules = [
+    "default-src 'none'",
+    "base-uri 'self'",
+    "frame-ancestors 'none'",
+    "style-src 'self' 'unsafe-inline'",
+    "font-src 'self'",
+    "manifest-src 'self'"
+  ]
+
+  @csp_start [Enum.join(static_csp_rules, ";") <> ";"]
+
   defp csp_string do
     scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme]
     static_url = Pleroma.Web.Endpoint.static_url()
     websocket_url = Pleroma.Web.Endpoint.websocket_url()
     report_uri = Config.get([:http_security, :report_uri])
 
-    connect_src = "connect-src 'self' #{static_url} #{websocket_url}"
+    img_src = "img-src 'self' data: blob:"
+    media_src = "media-src 'self'"
+
+    {img_src, media_src} =
+      if Config.get([:media_proxy, :enabled]) &&
+           !Config.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do
+        sources = get_proxy_and_attachment_sources()
+        {[img_src, sources], [media_src, sources]}
+      else
+        {[img_src, " https:"], [media_src, " https:"]}
+      end
+
+    connect_src = ["connect-src 'self' blob: ", static_url, ?\s, websocket_url]
 
     connect_src =
       if Pleroma.Config.get(:env) == :dev do
-        connect_src <> " http://localhost:3035/"
+        [connect_src, " http://localhost:3035/"]
       else
         connect_src
       end
@@ -71,27 +94,46 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do
         "script-src 'self'"
       end
 
-    main_part = [
+    report = if report_uri, do: ["report-uri ", report_uri, ";report-to csp-endpoint"]
-      "default-src 'none'",
+    insecure = if scheme == "https", do: "upgrade-insecure-requests"
-      "base-uri 'self'",
-      "frame-ancestors 'none'",
-      "img-src 'self' data: blob: https:",
-      "media-src 'self' https:",
-      "style-src 'self' 'unsafe-inline'",
-      "font-src 'self'",
-      "manifest-src 'self'",
-      connect_src,
-      script_src
-    ]
 
-    report = if report_uri, do: ["report-uri #{report_uri}; report-to csp-endpoint"], else: []
+    @csp_start
-
+    |> add_csp_param(img_src)
-    insecure = if scheme == "https", do: ["upgrade-insecure-requests"], else: []
+    |> add_csp_param(media_src)
-
+    |> add_csp_param(connect_src)
-    (main_part ++ report ++ insecure)
+    |> add_csp_param(script_src)
-    |> Enum.join("; ")
+    |> add_csp_param(insecure)
+    |> add_csp_param(report)
+    |> :erlang.iolist_to_binary()
   end
 
+  defp get_proxy_and_attachment_sources do
+    media_proxy_whitelist =
+      Enum.reduce(Config.get([:media_proxy, :whitelist]), [], fn host, acc ->
+        add_source(acc, host)
+      end)
+
+    upload_base_url =
+      if Config.get([Pleroma.Upload, :base_url]),
+        do: URI.parse(Config.get([Pleroma.Upload, :base_url])).host
+
+    s3_endpoint =
+      if Config.get([Pleroma.Upload, :uploader]) == Pleroma.Uploaders.S3,
+        do: URI.parse(Config.get([Pleroma.Uploaders.S3, :public_endpoint])).host
+
+    []
+    |> add_source(upload_base_url)
+    |> add_source(s3_endpoint)
+    |> add_source(media_proxy_whitelist)
+  end
+
+  defp add_source(iodata, nil), do: iodata
+  defp add_source(iodata, source), do: [[?\s, source] | iodata]
+
+  defp add_csp_param(csp_iodata, nil), do: csp_iodata
+
+  defp add_csp_param(csp_iodata, param), do: [[param, ?;] | csp_iodata]
+
   def warn_if_disabled do
     unless Config.get([:http_security, :enabled]) do
       Logger.warn("

lib/pleroma/user.ex

@@ -305,8 +305,13 @@ defmodule Pleroma.User do
 
   def avatar_url(user, options \\ []) do
     case user.avatar do
-      %{"url" => [%{"href" => href} | _]} -> href
+      %{"url" => [%{"href" => href} | _]} ->
-      _ -> !options[:no_default] && "#{Web.base_url()}/images/avi.png"
+        href
+
+      _ ->
+        unless options[:no_default] do
+          Config.get([:assets, :default_user_avatar], "#{Web.base_url()}/images/avi.png")
+        end
     end
   end
 
@@ -533,9 +538,10 @@ defmodule Pleroma.User do
     |> delete_change(:also_known_as)
     |> unique_constraint(:email)
     |> validate_format(:email, @email_regex)
+    |> validate_inclusion(:actor_type, ["Person", "Service"])
   end
 
-  @spec update_as_admin(%User{}, map) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()}
+  @spec update_as_admin(User.t(), map()) :: {:ok, User.t()} | {:error, Changeset.t()}
   def update_as_admin(user, params) do
     params = Map.put(params, "password_confirmation", params["password"])
     changeset = update_as_admin_changeset(user, params)
@@ -556,7 +562,7 @@ defmodule Pleroma.User do
     |> put_change(:password_reset_pending, false)
   end
 
-  @spec reset_password(User.t(), map) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()}
+  @spec reset_password(User.t(), map()) :: {:ok, User.t()} | {:error, Changeset.t()}
   def reset_password(%User{} = user, params) do
     reset_password(user, user, params)
   end
@@ -749,7 +755,19 @@ defmodule Pleroma.User do
     {:error, "Not subscribed!"}
   end
 
+  @spec unfollow(User.t(), User.t()) :: {:ok, User.t(), Activity.t()} | {:error, String.t()}
   def unfollow(%User{} = follower, %User{} = followed) do
+    case do_unfollow(follower, followed) do
+      {:ok, follower, followed} ->
+        {:ok, follower, Utils.fetch_latest_follow(follower, followed)}
+
+      error ->
+        error
+    end
+  end
+
+  @spec do_unfollow(User.t(), User.t()) :: {:ok, User.t(), User.t()} | {:error, String.t()}
+  defp do_unfollow(%User{} = follower, %User{} = followed) do
     case get_follow_state(follower, followed) do
       state when state in [:follow_pending, :follow_accept] ->
         FollowingRelationship.unfollow(follower, followed)
@@ -760,7 +778,7 @@ defmodule Pleroma.User do
           |> update_following_count()
           |> set_cache()
 
-        {:ok, follower, Utils.fetch_latest_follow(follower, followed)}
+        {:ok, follower, followed}
 
       nil ->
         {:error, "Not subscribed!"}
@@ -1191,8 +1209,9 @@ defmodule Pleroma.User do
 
   def increment_unread_conversation_count(_, user), do: {:ok, user}
 
-  @spec get_users_from_set([String.t()], boolean()) :: [User.t()]
+  @spec get_users_from_set([String.t()], keyword()) :: [User.t()]
-  def get_users_from_set(ap_ids, local_only \\ true) do
+  def get_users_from_set(ap_ids, opts \\ []) do
+    local_only = Keyword.get(opts, :local_only, true)
     criteria = %{ap_id: ap_ids, deactivated: false}
     criteria = if local_only, do: Map.put(criteria, :local, true), else: criteria
 
@@ -1204,7 +1223,9 @@ defmodule Pleroma.User do
   def get_recipients_from_activity(%Activity{recipients: to, actor: actor}) do
     to = [actor | to]
 
-    User.Query.build(%{recipients_from_activity: to, local: true, deactivated: false})
+    query = User.Query.build(%{recipients_from_activity: to, local: true, deactivated: false})
+
+    query
     |> Repo.all()
   end
 
@@ -1400,15 +1421,13 @@ defmodule Pleroma.User do
       user
       |> get_followers()
       |> Enum.filter(& &1.local)
-      |> Enum.each(fn follower ->
+      |> Enum.each(&set_cache(update_following_count(&1)))
-        follower |> update_following_count() |> set_cache()
-      end)
 
       # Only update local user counts, remote will be update during the next pull.
       user
       |> get_friends()
       |> Enum.filter(& &1.local)
-      |> Enum.each(&update_follower_count/1)
+      |> Enum.each(&do_unfollow(user, &1))
 
       {:ok, user}
     end
@@ -1430,6 +1449,25 @@ defmodule Pleroma.User do
     BackgroundWorker.enqueue("delete_user", %{"user_id" => user.id})
   end
 
+  defp delete_and_invalidate_cache(%User{} = user) do
+    invalidate_cache(user)
+    Repo.delete(user)
+  end
+
+  defp delete_or_deactivate(%User{local: false} = user), do: delete_and_invalidate_cache(user)
+
+  defp delete_or_deactivate(%User{local: true} = user) do
+    status = account_status(user)
+
+    if status == :confirmation_pending do
+      delete_and_invalidate_cache(user)
+    else
+      user
+      |> change(%{deactivated: true, email: nil})
+      |> update_and_set_cache()
+    end
+  end
+
   def perform(:force_password_reset, user), do: force_password_reset(user)
 
   @spec perform(atom(), User.t()) :: {:ok, User.t()}
@@ -1451,14 +1489,9 @@ defmodule Pleroma.User do
 
     delete_user_activities(user)
 
-    if user.local do
+    delete_outgoing_pending_follow_requests(user)
-      user
+
-      |> change(%{deactivated: true, email: nil})
+    delete_or_deactivate(user)
-      |> update_and_set_cache()
-    else
-      invalidate_cache(user)
-      Repo.delete(user)
-    end
   end
 
   def perform(:deactivate_async, user, status), do: deactivate(user, status)
@@ -1580,6 +1613,12 @@ defmodule Pleroma.User do
 
   defp delete_activity(_activity, _user), do: "Doing nothing"
 
+  defp delete_outgoing_pending_follow_requests(user) do
+    user
+    |> FollowingRelationship.outgoing_pending_follow_requests_query()
+    |> Repo.delete_all()
+  end
+
   def html_filter_policy(%User{no_rich_text: true}) do
     Pleroma.HTML.Scrubber.TwitterText
   end
@@ -1589,12 +1628,19 @@ defmodule Pleroma.User do
   def fetch_by_ap_id(ap_id), do: ActivityPub.make_user_from_ap_id(ap_id)
 
   def get_or_fetch_by_ap_id(ap_id) do
-    user = get_cached_by_ap_id(ap_id)
+    cached_user = get_cached_by_ap_id(ap_id)
 
-    if !is_nil(user) and !needs_update?(user) do
+    maybe_fetched_user = needs_update?(cached_user) && fetch_by_ap_id(ap_id)
-      {:ok, user}
+
-    else
+    case {cached_user, maybe_fetched_user} do
-      fetch_by_ap_id(ap_id)
+      {_, {:ok, %User{} = user}} ->
+        {:ok, user}
+
+      {%User{} = user, _} ->
+        {:ok, user}
+
+      _ ->
+        {:error, :not_found}
     end
   end
 

lib/pleroma/user/query.ex

@@ -45,7 +45,7 @@ defmodule Pleroma.User.Query do
             is_admin: boolean(),
             is_moderator: boolean(),
             super_users: boolean(),
-            exclude_service_users: boolean(),
+            invisible: boolean(),
             followers: User.t(),
             friends: User.t(),
             recipients_from_activity: [String.t()],
@@ -89,8 +89,8 @@ defmodule Pleroma.User.Query do
     where(query, [u], ilike(field(u, ^key), ^"%#{value}%"))
   end
 
-  defp compose_query({:exclude_service_users, _}, query) do
+  defp compose_query({:invisible, bool}, query) when is_boolean(bool) do
-    where(query, [u], not like(u.ap_id, "%/relay") and not like(u.ap_id, "%/internal/fetch"))
+    where(query, [u], u.invisible == ^bool)
   end
 
   defp compose_query({key, value}, query)
@@ -167,20 +167,18 @@ defmodule Pleroma.User.Query do
   end
 
   defp compose_query({:recipients_from_activity, to}, query) do
-    query
+    following_query =
-    |> join(:left, [u], r in FollowingRelationship,
+      from(u in User,
-      as: :relationships,
+        join: f in FollowingRelationship,
-      on: r.follower_id == u.id
+        on: u.id == f.following_id,
+        where: f.state == ^:follow_accept,
+        where: u.follower_address in ^to,
+        select: f.follower_id
+      )
+
+    from(u in query,
+      where: u.ap_id in ^to or u.id in subquery(following_query)
     )
-    |> join(:left, [relationships: r], f in User,
-      as: :following,
-      on: f.id == r.following_id
-    )
-    |> where(
-      [u, following: f, relationships: r],
-      u.ap_id in ^to or (f.follower_address in ^to and r.state == ^:follow_accept)
-    )
-    |> distinct(true)
   end
 
   defp compose_query({:order_by, key}, query) do

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.Constants
   alias Pleroma.Conversation
   alias Pleroma.Conversation.Participation
+  alias Pleroma.Maps
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Object.Containment
@@ -19,7 +20,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.MRF
   alias Pleroma.Web.ActivityPub.Transmogrifier
-  alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.Streamer
   alias Pleroma.Web.WebFinger
   alias Pleroma.Workers.BackgroundWorker
@@ -67,16 +67,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     {recipients, to, cc}
   end
 
-  defp check_actor_is_active(actor) do
+  defp check_actor_is_active(nil), do: true
-    if not is_nil(actor) do
+
-      with user <- User.get_cached_by_ap_id(actor),
+  defp check_actor_is_active(actor) when is_binary(actor) do
-           false <- user.deactivated do
+    case User.get_cached_by_ap_id(actor) do
-        true
+      %User{deactivated: deactivated} -> not deactivated
-      else
+      _ -> false
-        _e -> false
-      end
-    else
-      true
     end
   end
 
@@ -87,7 +83,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp check_remote_limit(_), do: true
 
-  def increase_note_count_if_public(actor, object) do
+  defp increase_note_count_if_public(actor, object) do
     if is_public?(object), do: User.increase_note_count(actor), else: {:ok, actor}
   end
 
@@ -95,36 +91,26 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     if is_public?(object), do: User.decrease_note_count(actor), else: {:ok, actor}
   end
 
-  def increase_replies_count_if_reply(%{
+  defp increase_replies_count_if_reply(%{
-        "object" => %{"inReplyTo" => reply_ap_id} = object,
+         "object" => %{"inReplyTo" => reply_ap_id} = object,
-        "type" => "Create"
+         "type" => "Create"
-      }) do
+       }) do
     if is_public?(object) do
       Object.increase_replies_count(reply_ap_id)
     end
   end
 
-  def increase_replies_count_if_reply(_create_data), do: :noop
+  defp increase_replies_count_if_reply(_create_data), do: :noop
 
-  def decrease_replies_count_if_reply(%Object{
+  defp increase_poll_votes_if_vote(%{
-        data: %{"inReplyTo" => reply_ap_id} = object
+         "object" => %{"inReplyTo" => reply_ap_id, "name" => name},
-      }) do
+         "type" => "Create",
-    if is_public?(object) do
+         "actor" => actor
-      Object.decrease_replies_count(reply_ap_id)
+       }) do
-    end
-  end
-
-  def decrease_replies_count_if_reply(_object), do: :noop
-
-  def increase_poll_votes_if_vote(%{
-        "object" => %{"inReplyTo" => reply_ap_id, "name" => name},
-        "type" => "Create",
-        "actor" => actor
-      }) do
     Object.increase_vote_count(reply_ap_id, name, actor)
   end
 
-  def increase_poll_votes_if_vote(_create_data), do: :noop
+  defp increase_poll_votes_if_vote(_create_data), do: :noop
 
   @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
   def persist(object, meta) do
@@ -161,12 +147,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
         })
 
       # Splice in the child object if we have one.
-      activity =
+      activity = Maps.put_if_present(activity, :object, object)
-        if not is_nil(object) do
-          Map.put(activity, :object, object)
-        else
-          activity
-        end
 
       BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id})
 
@@ -203,8 +184,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp create_or_bump_conversation(activity, actor) do
     with {:ok, conversation} <- Conversation.create_or_bump_for(activity),
-         %User{} = user <- User.get_cached_by_ap_id(actor),
+         %User{} = user <- User.get_cached_by_ap_id(actor) do
-         Participation.mark_as_read(user, conversation) do
+      Participation.mark_as_read(user, conversation)
       {:ok, conversation}
     end
   end
@@ -226,13 +207,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   def stream_out_participations(%Object{data: %{"context" => context}}, user) do
-    with %Conversation{} = conversation <- Conversation.get_for_ap_id(context),
+    with %Conversation{} = conversation <- Conversation.get_for_ap_id(context) do
-         conversation = Repo.preload(conversation, :participations),
+      conversation = Repo.preload(conversation, :participations)
-         last_activity_id =
+
-           fetch_latest_activity_id_for_context(conversation.ap_id, %{
+      last_activity_id =
-             "user" => user,
+        fetch_latest_activity_id_for_context(conversation.ap_id, %{
-             "blocking_user" => user
+          user: user,
-           }) do
+          blocking_user: user
+        })
+
       if last_activity_id do
         stream_out_participations(conversation.participations)
       end
@@ -266,12 +249,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     published = params[:published]
     quick_insert? = Config.get([:env]) == :benchmark
 
-    with create_data <-
+    create_data =
-           make_create_data(
+      make_create_data(
-             %{to: to, actor: actor, published: published, context: context, object: object},
+        %{to: to, actor: actor, published: published, context: context, object: object},
-             additional
+        additional
-           ),
+      )
-         {:ok, activity} <- insert(create_data, local, fake),
+
+    with {:ok, activity} <- insert(create_data, local, fake),
          {:fake, false, activity} <- {:fake, fake, activity},
          _ <- increase_replies_count_if_reply(create_data),
          _ <- increase_poll_votes_if_vote(create_data),
@@ -299,12 +283,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     local = !(params[:local] == false)
     published = params[:published]
 
-    with listen_data <-
+    listen_data =
-           make_listen_data(
+      make_listen_data(
-             %{to: to, actor: actor, published: published, context: context, object: object},
+        %{to: to, actor: actor, published: published, context: context, object: object},
-             additional
+        additional
-           ),
+      )
-         {:ok, activity} <- insert(listen_data, local),
+
+    with {:ok, activity} <- insert(listen_data, local),
          _ <- notify_and_stream(activity),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
@@ -322,14 +307,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   @spec accept_or_reject(String.t(), map()) :: {:ok, Activity.t()} | {:error, any()}
-  def accept_or_reject(type, %{to: to, actor: actor, object: object} = params) do
+  defp accept_or_reject(type, %{to: to, actor: actor, object: object} = params) do
     local = Map.get(params, :local, true)
     activity_id = Map.get(params, :activity_id, nil)
 
-    with data <-
+    data =
-           %{"to" => to, "type" => type, "actor" => actor.ap_id, "object" => object}
+      %{"to" => to, "type" => type, "actor" => actor.ap_id, "object" => object}
-           |> Utils.maybe_put("id", activity_id),
+      |> Maps.put_if_present("id", activity_id)
-         {:ok, activity} <- insert(data, local),
+
+    with {:ok, activity} <- insert(data, local),
          _ <- notify_and_stream(activity),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
@@ -341,51 +327,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     local = !(params[:local] == false)
     activity_id = params[:activity_id]
 
-    with data <- %{
+    data =
-           "to" => to,
+      %{
-           "cc" => cc,
+        "to" => to,
-           "type" => "Update",
+        "cc" => cc,
-           "actor" => actor,
+        "type" => "Update",
-           "object" => object
+        "actor" => actor,
-         },
+        "object" => object
-         data <- Utils.maybe_put(data, "id", activity_id),
+      }
-         {:ok, activity} <- insert(data, local),
+      |> Maps.put_if_present("id", activity_id)
+
+    with {:ok, activity} <- insert(data, local),
          _ <- notify_and_stream(activity),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
     end
   end
 
-  @spec announce(User.t(), Object.t(), String.t() | nil, boolean(), boolean()) ::
-          {:ok, Activity.t(), Object.t()} | {:error, any()}
-  def announce(
-        %User{ap_id: _} = user,
-        %Object{data: %{"id" => _}} = object,
-        activity_id \\ nil,
-        local \\ true,
-        public \\ true
-      ) do
-    with {:ok, result} <-
-           Repo.transaction(fn -> do_announce(user, object, activity_id, local, public) end) do
-      result
-    end
-  end
-
-  defp do_announce(user, object, activity_id, local, public) do
-    with true <- is_announceable?(object, user, public),
-         object <- Object.get_by_id(object.id),
-         announce_data <- make_announce_data(user, object, activity_id, public),
-         {:ok, activity} <- insert(announce_data, local),
-         {:ok, object} <- add_announce_to_object(activity, object),
-         _ <- notify_and_stream(activity),
-         :ok <- maybe_federate(activity) do
-      {:ok, activity, object}
-    else
-      false -> {:error, false}
-      {:error, error} -> Repo.rollback(error)
-    end
-  end
-
   @spec follow(User.t(), User.t(), String.t() | nil, boolean()) ::
           {:ok, Activity.t()} | {:error, any()}
   def follow(follower, followed, activity_id \\ nil, local \\ true) do
@@ -396,8 +354,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   defp do_follow(follower, followed, activity_id, local) do
-    with data <- make_follow_data(follower, followed, activity_id),
+    data = make_follow_data(follower, followed, activity_id)
-         {:ok, activity} <- insert(data, local),
+
+    with {:ok, activity} <- insert(data, local),
          _ <- notify_and_stream(activity),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
@@ -441,13 +400,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp do_block(blocker, blocked, activity_id, local) do
     unfollow_blocked = Config.get([:activitypub, :unfollow_blocked])
 
-    if unfollow_blocked do
+    if unfollow_blocked and fetch_latest_follow(blocker, blocked) do
-      follow_activity = fetch_latest_follow(blocker, blocked)
+      unfollow(blocker, blocked, nil, local)
-      if follow_activity, do: unfollow(blocker, blocked, nil, local)
     end
 
-    with block_data <- make_block_data(blocker, blocked, activity_id),
+    block_data = make_block_data(blocker, blocked, activity_id)
-         {:ok, activity} <- insert(block_data, local),
+
+    with {:ok, activity} <- insert(block_data, local),
          _ <- notify_and_stream(activity),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
@@ -526,8 +485,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     public = [Constants.as_public()]
 
     recipients =
-      if opts["user"],
+      if opts[:user],
-        do: [opts["user"].ap_id | User.following(opts["user"])] ++ public,
+        do: [opts[:user].ap_id | User.following(opts[:user])] ++ public,
         else: public
 
     from(activity in Activity)
@@ -535,7 +494,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> maybe_preload_bookmarks(opts)
     |> maybe_set_thread_muted_field(opts)
     |> restrict_blocked(opts)
-    |> restrict_recipients(recipients, opts["user"])
+    |> restrict_recipients(recipients, opts[:user])
     |> where(
       [activity],
       fragment(
@@ -562,41 +521,45 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           FlakeId.Ecto.CompatType.t() | nil
   def fetch_latest_activity_id_for_context(context, opts \\ %{}) do
     context
-    |> fetch_activities_for_context_query(Map.merge(%{"skip_preload" => true}, opts))
+    |> fetch_activities_for_context_query(Map.merge(%{skip_preload: true}, opts))
     |> limit(1)
     |> select([a], a.id)
     |> Repo.one()
   end
 
-  @spec fetch_public_activities(map(), Pagination.type()) :: [Activity.t()]
+  @spec fetch_public_or_unlisted_activities(map(), Pagination.type()) :: [Activity.t()]
-  def fetch_public_activities(opts \\ %{}, pagination \\ :keyset) do
+  def fetch_public_or_unlisted_activities(opts \\ %{}, pagination \\ :keyset) do
-    opts = Map.drop(opts, ["user"])
+    opts = Map.delete(opts, :user)
 
     [Constants.as_public()]
     |> fetch_activities_query(opts)
-    |> restrict_unlisted()
+    |> restrict_unlisted(opts)
     |> Pagination.fetch_paginated(opts, pagination)
   end
 
+  @spec fetch_public_activities(map(), Pagination.type()) :: [Activity.t()]
+  def fetch_public_activities(opts \\ %{}, pagination \\ :keyset) do
+    opts
+    |> Map.put(:restrict_unlisted, true)
+    |> fetch_public_or_unlisted_activities(pagination)
+  end
+
   @valid_visibilities ~w[direct unlisted public private]
 
   defp restrict_visibility(query, %{visibility: visibility})
        when is_list(visibility) do
     if Enum.all?(visibility, &(&1 in @valid_visibilities)) do
-      query =
+      from(
-        from(
+        a in query,
-          a in query,
+        where:
-          where:
+          fragment(
-            fragment(
+            "activity_visibility(?, ?, ?) = ANY (?)",
-              "activity_visibility(?, ?, ?) = ANY (?)",
+            a.actor,
-              a.actor,
+            a.recipients,
-              a.recipients,
+            a.data,
-              a.data,
+            ^visibility
-              ^visibility
+          )
-            )
+      )
-        )
-
-      query
     else
       Logger.error("Could not restrict visibility to #{visibility}")
     end
@@ -618,7 +581,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_visibility(query, _visibility), do: query
 
-  defp exclude_visibility(query, %{"exclude_visibilities" => visibility})
+  defp exclude_visibility(query, %{exclude_visibilities: visibility})
        when is_list(visibility) do
     if Enum.all?(visibility, &(&1 in @valid_visibilities)) do
       from(
@@ -638,7 +601,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  defp exclude_visibility(query, %{"exclude_visibilities" => visibility})
+  defp exclude_visibility(query, %{exclude_visibilities: visibility})
        when visibility in @valid_visibilities do
     from(
       a in query,
@@ -653,7 +616,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     )
   end
 
-  defp exclude_visibility(query, %{"exclude_visibilities" => visibility})
+  defp exclude_visibility(query, %{exclude_visibilities: visibility})
        when visibility not in [nil | @valid_visibilities] do
     Logger.error("Could not exclude visibility to #{visibility}")
     query
@@ -664,14 +627,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp restrict_thread_visibility(query, _, %{skip_thread_containment: true} = _),
     do: query
 
-  defp restrict_thread_visibility(
+  defp restrict_thread_visibility(query, %{user: %User{skip_thread_containment: true}}, _),
-         query,
+    do: query
-         %{"user" => %User{skip_thread_containment: true}},
-         _
-       ),
-       do: query
 
-  defp restrict_thread_visibility(query, %{"user" => %User{ap_id: ap_id}}, _) do
+  defp restrict_thread_visibility(query, %{user: %User{ap_id: ap_id}}, _) do
     from(
       a in query,
       where: fragment("thread_visibility(?, (?)->>'id') = true", ^ap_id, a.data)
@@ -683,87 +642,79 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   def fetch_user_abstract_activities(user, reading_user, params \\ %{}) do
     params =
       params
-      |> Map.put("user", reading_user)
+      |> Map.put(:user, reading_user)
-      |> Map.put("actor_id", user.ap_id)
+      |> Map.put(:actor_id, user.ap_id)
 
-    recipients =
+    %{
-      user_activities_recipients(%{
+      godmode: params[:godmode],
-        "godmode" => params["godmode"],
+      reading_user: reading_user
-        "reading_user" => reading_user
+    }
-      })
+    |> user_activities_recipients()
-
+    |> fetch_activities(params)
-    fetch_activities(recipients, params)
     |> Enum.reverse()
   end
 
   def fetch_user_activities(user, reading_user, params \\ %{}) do
     params =
       params
-      |> Map.put("type", ["Create", "Announce"])
+      |> Map.put(:type, ["Create", "Announce"])
-      |> Map.put("user", reading_user)
+      |> Map.put(:user, reading_user)
-      |> Map.put("actor_id", user.ap_id)
+      |> Map.put(:actor_id, user.ap_id)
-      |> Map.put("pinned_activity_ids", user.pinned_activities)
+      |> Map.put(:pinned_activity_ids, user.pinned_activities)
 
     params =
       if User.blocks?(reading_user, user) do
         params
       else
         params
-        |> Map.put("blocking_user", reading_user)
+        |> Map.put(:blocking_user, reading_user)
-        |> Map.put("muting_user", reading_user)
+        |> Map.put(:muting_user, reading_user)
       end
 
-    recipients =
+    %{
-      user_activities_recipients(%{
+      godmode: params[:godmode],
-        "godmode" => params["godmode"],
+      reading_user: reading_user
-        "reading_user" => reading_user
+    }
-      })
+    |> user_activities_recipients()
-
+    |> fetch_activities(params)
-    fetch_activities(recipients, params)
     |> Enum.reverse()
   end
 
   def fetch_statuses(reading_user, params) do
-    params =
+    params = Map.put(params, :type, ["Create", "Announce"])
-      params
-      |> Map.put("type", ["Create", "Announce"])
 
-    recipients =
+    %{
-      user_activities_recipients(%{
+      godmode: params[:godmode],
-        "godmode" => params["godmode"],
+      reading_user: reading_user
-        "reading_user" => reading_user
+    }
-      })
+    |> user_activities_recipients()
-
+    |> fetch_activities(params, :offset)
-    fetch_activities(recipients, params, :offset)
     |> Enum.reverse()
   end
 
-  defp user_activities_recipients(%{"godmode" => true}) do
+  defp user_activities_recipients(%{godmode: true}), do: []
-    []
-  end
 
-  defp user_activities_recipients(%{"reading_user" => reading_user}) do
+  defp user_activities_recipients(%{reading_user: reading_user}) do
     if reading_user do
-      [Constants.as_public()] ++ [reading_user.ap_id | User.following(reading_user)]
+      [Constants.as_public(), reading_user.ap_id | User.following(reading_user)]
     else
       [Constants.as_public()]
     end
   end
 
-  defp restrict_since(query, %{"since_id" => ""}), do: query
+  defp restrict_since(query, %{since_id: ""}), do: query
 
-  defp restrict_since(query, %{"since_id" => since_id}) do
+  defp restrict_since(query, %{since_id: since_id}) do
     from(activity in query, where: activity.id > ^since_id)
   end
 
   defp restrict_since(query, _), do: query
 
-  defp restrict_tag_reject(_query, %{"tag_reject" => _tag_reject, "skip_preload" => true}) do
+  defp restrict_tag_reject(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
     raise "Can't use the child object without preloading!"
   end
 
-  defp restrict_tag_reject(query, %{"tag_reject" => tag_reject})
+  defp restrict_tag_reject(query, %{tag_reject: [_ | _] = tag_reject}) do
-       when is_list(tag_reject) and tag_reject != [] do
     from(
       [_activity, object] in query,
       where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject)
@@ -772,12 +723,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_tag_reject(query, _), do: query
 
-  defp restrict_tag_all(_query, %{"tag_all" => _tag_all, "skip_preload" => true}) do
+  defp restrict_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do
     raise "Can't use the child object without preloading!"
   end
 
-  defp restrict_tag_all(query, %{"tag_all" => tag_all})
+  defp restrict_tag_all(query, %{tag_all: [_ | _] = tag_all}) do
-       when is_list(tag_all) and tag_all != [] do
     from(
       [_activity, object] in query,
       where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all)
@@ -786,18 +736,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_tag_all(query, _), do: query
 
-  defp restrict_tag(_query, %{"tag" => _tag, "skip_preload" => true}) do
+  defp restrict_tag(_query, %{tag: _tag, skip_preload: true}) do
     raise "Can't use the child object without preloading!"
   end
 
-  defp restrict_tag(query, %{"tag" => tag}) when is_list(tag) do
+  defp restrict_tag(query, %{tag: tag}) when is_list(tag) do
     from(
       [_activity, object] in query,
       where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag)
     )
   end
 
-  defp restrict_tag(query, %{"tag" => tag}) when is_binary(tag) do
+  defp restrict_tag(query, %{tag: tag}) when is_binary(tag) do
     from(
       [_activity, object] in query,
       where: fragment("(?)->'tag' \\? (?)", object.data, ^tag)
@@ -820,35 +770,35 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     )
   end
 
-  defp restrict_local(query, %{"local_only" => true}) do
+  defp restrict_local(query, %{local_only: true}) do
     from(activity in query, where: activity.local == true)
   end
 
   defp restrict_local(query, _), do: query
 
-  defp restrict_actor(query, %{"actor_id" => actor_id}) do
+  defp restrict_actor(query, %{actor_id: actor_id}) do
     from(activity in query, where: activity.actor == ^actor_id)
   end
 
   defp restrict_actor(query, _), do: query
 
-  defp restrict_type(query, %{"type" => type}) when is_binary(type) do
+  defp restrict_type(query, %{type: type}) when is_binary(type) do
     from(activity in query, where: fragment("?->>'type' = ?", activity.data, ^type))
   end
 
-  defp restrict_type(query, %{"type" => type}) do
+  defp restrict_type(query, %{type: type}) do
     from(activity in query, where: fragment("?->>'type' = ANY(?)", activity.data, ^type))
   end
 
   defp restrict_type(query, _), do: query
 
-  defp restrict_state(query, %{"state" => state}) do
+  defp restrict_state(query, %{state: state}) do
     from(activity in query, where: fragment("?->>'state' = ?", activity.data, ^state))
   end
 
   defp restrict_state(query, _), do: query
 
-  defp restrict_favorited_by(query, %{"favorited_by" => ap_id}) do
+  defp restrict_favorited_by(query, %{favorited_by: ap_id}) do
     from(
       [_activity, object] in query,
       where: fragment("(?)->'likes' \\? (?)", object.data, ^ap_id)
@@ -857,11 +807,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_favorited_by(query, _), do: query
 
-  defp restrict_media(_query, %{"only_media" => _val, "skip_preload" => true}) do
+  defp restrict_media(_query, %{only_media: _val, skip_preload: true}) do
     raise "Can't use the child object without preloading!"
   end
 
-  defp restrict_media(query, %{"only_media" => val}) when val in [true, "true", "1"] do
+  defp restrict_media(query, %{only_media: true}) do
     from(
       [_activity, object] in query,
       where: fragment("not (?)->'attachment' = (?)", object.data, ^[])
@@ -870,7 +820,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_media(query, _), do: query
 
-  defp restrict_replies(query, %{"exclude_replies" => val}) when val in [true, "true", "1"] do
+  defp restrict_replies(query, %{exclude_replies: true}) do
     from(
       [_activity, object] in query,
       where: fragment("?->>'inReplyTo' is null", object.data)
@@ -878,8 +828,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   defp restrict_replies(query, %{
-         "reply_filtering_user" => user,
+         reply_filtering_user: user,
-         "reply_visibility" => "self"
+         reply_visibility: "self"
        }) do
     from(
       [activity, object] in query,
@@ -894,8 +844,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   defp restrict_replies(query, %{
-         "reply_filtering_user" => user,
+         reply_filtering_user: user,
-         "reply_visibility" => "following"
+         reply_visibility: "following"
        }) do
     from(
       [activity, object] in query,
@@ -914,16 +864,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_replies(query, _), do: query
 
-  defp restrict_reblogs(query, %{"exclude_reblogs" => val}) when val in [true, "true", "1"] do
+  defp restrict_reblogs(query, %{exclude_reblogs: true}) do
     from(activity in query, where: fragment("?->>'type' != 'Announce'", activity.data))
   end
 
   defp restrict_reblogs(query, _), do: query
 
-  defp restrict_muted(query, %{"with_muted" => val}) when val in [true, "true", "1"], do: query
+  defp restrict_muted(query, %{with_muted: true}), do: query
 
-  defp restrict_muted(query, %{"muting_user" => %User{} = user} = opts) do
+  defp restrict_muted(query, %{muting_user: %User{} = user} = opts) do
-    mutes = opts["muted_users_ap_ids"] || User.muted_users_ap_ids(user)
+    mutes = opts[:muted_users_ap_ids] || User.muted_users_ap_ids(user)
 
     query =
       from([activity] in query,
@@ -931,7 +881,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
         where: fragment("not (?->'to' \\?| ?)", activity.data, ^mutes)
       )
 
-    unless opts["skip_preload"] do
+    unless opts[:skip_preload] do
       from([thread_mute: tm] in query, where: is_nil(tm.user_id))
     else
       query
@@ -940,8 +890,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_muted(query, _), do: query
 
-  defp restrict_blocked(query, %{"blocking_user" => %User{} = user} = opts) do
+  defp restrict_blocked(query, %{blocking_user: %User{} = user} = opts) do
-    blocked_ap_ids = opts["blocked_users_ap_ids"] || User.blocked_users_ap_ids(user)
+    blocked_ap_ids = opts[:blocked_users_ap_ids] || User.blocked_users_ap_ids(user)
     domain_blocks = user.domain_blocks || []
 
     following_ap_ids = User.get_friends_ap_ids(user)
@@ -953,6 +903,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       [activity, object: o] in query,
       where: fragment("not (? = ANY(?))", activity.actor, ^blocked_ap_ids),
       where: fragment("not (? && ?)", activity.recipients, ^blocked_ap_ids),
+      where:
+        fragment(
+          "recipients_contain_blocked_domains(?, ?) = false",
+          activity.recipients,
+          ^domain_blocks
+        ),
       where:
         fragment(
           "not (?->>'type' = 'Announce' and ?->'to' \\?| ?)",
@@ -981,7 +937,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_blocked(query, _), do: query
 
-  defp restrict_unlisted(query) do
+  defp restrict_unlisted(query, %{restrict_unlisted: true}) do
     from(
       activity in query,
       where:
@@ -993,19 +949,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     )
   end
 
-  # TODO: when all endpoints migrated to OpenAPI compare `pinned` with `true` (boolean) only,
+  defp restrict_unlisted(query, _), do: query
-  # the same for `restrict_media/2`, `restrict_replies/2`, 'restrict_reblogs/2'
-  # and `restrict_muted/2`
 
-  defp restrict_pinned(query, %{"pinned" => pinned, "pinned_activity_ids" => ids})
+  defp restrict_pinned(query, %{pinned: true, pinned_activity_ids: ids}) do
-       when pinned in [true, "true", "1"] do
     from(activity in query, where: activity.id in ^ids)
   end
 
   defp restrict_pinned(query, _), do: query
 
-  defp restrict_muted_reblogs(query, %{"muting_user" => %User{} = user} = opts) do
+  defp restrict_muted_reblogs(query, %{muting_user: %User{} = user} = opts) do
-    muted_reblogs = opts["reblog_muted_users_ap_ids"] || User.reblog_muted_users_ap_ids(user)
+    muted_reblogs = opts[:reblog_muted_users_ap_ids] || User.reblog_muted_users_ap_ids(user)
 
     from(
       activity in query,
@@ -1021,7 +974,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_muted_reblogs(query, _), do: query
 
-  defp restrict_instance(query, %{"instance" => instance}) do
+  defp restrict_instance(query, %{instance: instance}) do
     users =
       from(
         u in User,
@@ -1035,7 +988,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_instance(query, _), do: query
 
-  defp exclude_poll_votes(query, %{"include_poll_votes" => true}), do: query
+  defp exclude_poll_votes(query, %{include_poll_votes: true}), do: query
 
   defp exclude_poll_votes(query, _) do
     if has_named_binding?(query, :object) do
@@ -1047,38 +1000,49 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  defp exclude_id(query, %{"exclude_id" => id}) when is_binary(id) do
+  defp exclude_invisible_actors(query, %{invisible_actors: true}), do: query
+
+  defp exclude_invisible_actors(query, _opts) do
+    invisible_ap_ids =
+      User.Query.build(%{invisible: true, select: [:ap_id]})
+      |> Repo.all()
+      |> Enum.map(fn %{ap_id: ap_id} -> ap_id end)
+
+    from([activity] in query, where: activity.actor not in ^invisible_ap_ids)
+  end
+
+  defp exclude_id(query, %{exclude_id: id}) when is_binary(id) do
     from(activity in query, where: activity.id != ^id)
   end
 
   defp exclude_id(query, _), do: query
 
-  defp maybe_preload_objects(query, %{"skip_preload" => true}), do: query
+  defp maybe_preload_objects(query, %{skip_preload: true}), do: query
 
   defp maybe_preload_objects(query, _) do
     query
     |> Activity.with_preloaded_object()
   end
 
-  defp maybe_preload_bookmarks(query, %{"skip_preload" => true}), do: query
+  defp maybe_preload_bookmarks(query, %{skip_preload: true}), do: query
 
   defp maybe_preload_bookmarks(query, opts) do
     query
-    |> Activity.with_preloaded_bookmark(opts["user"])
+    |> Activity.with_preloaded_bookmark(opts[:user])
   end
 
-  defp maybe_preload_report_notes(query, %{"preload_report_notes" => true}) do
+  defp maybe_preload_report_notes(query, %{preload_report_notes: true}) do
     query
     |> Activity.with_preloaded_report_notes()
   end
 
   defp maybe_preload_report_notes(query, _), do: query
 
-  defp maybe_set_thread_muted_field(query, %{"skip_preload" => true}), do: query
+  defp maybe_set_thread_muted_field(query, %{skip_preload: true}), do: query
 
   defp maybe_set_thread_muted_field(query, opts) do
     query
-    |> Activity.with_set_thread_muted_field(opts["muting_user"] || opts["user"])
+    |> Activity.with_set_thread_muted_field(opts[:muting_user] || opts[:user])
   end
 
   defp maybe_order(query, %{order: :desc}) do
@@ -1094,24 +1058,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp maybe_order(query, _), do: query
 
   defp fetch_activities_query_ap_ids_ops(opts) do
-    source_user = opts["muting_user"]
+    source_user = opts[:muting_user]
     ap_id_relationships = if source_user, do: [:mute, :reblog_mute], else: []
 
     ap_id_relationships =
-      ap_id_relationships ++
+      if opts[:blocking_user] && opts[:blocking_user] == source_user do
-        if opts["blocking_user"] && opts["blocking_user"] == source_user do
+        [:block | ap_id_relationships]
-          [:block]
+      else
-        else
+        ap_id_relationships
-          []
+      end
-        end
 
     preloaded_ap_ids = User.outgoing_relationships_ap_ids(source_user, ap_id_relationships)
 
-    restrict_blocked_opts = Map.merge(%{"blocked_users_ap_ids" => preloaded_ap_ids[:block]}, opts)
+    restrict_blocked_opts = Map.merge(%{blocked_users_ap_ids: preloaded_ap_ids[:block]}, opts)
-    restrict_muted_opts = Map.merge(%{"muted_users_ap_ids" => preloaded_ap_ids[:mute]}, opts)
+    restrict_muted_opts = Map.merge(%{muted_users_ap_ids: preloaded_ap_ids[:mute]}, opts)
 
     restrict_muted_reblogs_opts =
-      Map.merge(%{"reblog_muted_users_ap_ids" => preloaded_ap_ids[:reblog_mute]}, opts)
+      Map.merge(%{reblog_muted_users_ap_ids: preloaded_ap_ids[:reblog_mute]}, opts)
 
     {restrict_blocked_opts, restrict_muted_opts, restrict_muted_reblogs_opts}
   end
@@ -1130,7 +1093,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> maybe_preload_report_notes(opts)
     |> maybe_set_thread_muted_field(opts)
     |> maybe_order(opts)
-    |> restrict_recipients(recipients, opts["user"])
+    |> restrict_recipients(recipients, opts[:user])
     |> restrict_replies(opts)
     |> restrict_tag(opts)
     |> restrict_tag_reject(opts)
@@ -1152,16 +1115,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> restrict_instance(opts)
     |> Activity.restrict_deactivated_users()
     |> exclude_poll_votes(opts)
+    |> exclude_invisible_actors(opts)
     |> exclude_visibility(opts)
   end
 
   def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do
-    list_memberships = Pleroma.List.memberships(opts["user"])
+    list_memberships = Pleroma.List.memberships(opts[:user])
 
     fetch_activities_query(recipients ++ list_memberships, opts)
     |> Pagination.fetch_paginated(opts, pagination)
     |> Enum.reverse()
-    |> maybe_update_cc(list_memberships, opts["user"])
+    |> maybe_update_cc(list_memberships, opts[:user])
   end
 
   @doc """
@@ -1175,18 +1139,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> Activity.with_joined_object()
     |> Object.with_joined_activity()
     |> select([_like, object, activity], %{activity | object: object})
-    |> order_by([like, _, _], desc: like.id)
+    |> order_by([like, _, _], desc_nulls_last: like.id)
     |> Pagination.fetch_paginated(
-      Map.merge(params, %{"skip_order" => true}),
+      Map.merge(params, %{skip_order: true}),
       pagination,
       :object_activity
     )
   end
 
-  defp maybe_update_cc(activities, list_memberships, %User{ap_id: user_ap_id})
+  defp maybe_update_cc(activities, [_ | _] = list_memberships, %User{ap_id: user_ap_id}) do
-       when is_list(list_memberships) and length(list_memberships) > 0 do
     Enum.map(activities, fn
-      %{data: %{"bcc" => bcc}} = activity when is_list(bcc) and length(bcc) > 0 ->
+      %{data: %{"bcc" => [_ | _] = bcc}} = activity ->
         if Enum.any?(bcc, &(&1 in list_memberships)) do
           update_in(activity.data["cc"], &[user_ap_id | &1])
         else
@@ -1200,7 +1163,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp maybe_update_cc(activities, _, _), do: activities
 
-  def fetch_activities_bounded_query(query, recipients, recipients_with_public) do
+  defp fetch_activities_bounded_query(query, recipients, recipients_with_public) do
     from(activity in query,
       where:
         fragment("? && ?", activity.recipients, ^recipients) or
@@ -1224,12 +1187,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   @spec upload(Upload.source(), keyword()) :: {:ok, Object.t()} | {:error, any()}
   def upload(file, opts \\ []) do
     with {:ok, data} <- Upload.store(file, opts) do
-      obj_data =
+      obj_data = Maps.put_if_present(data, "actor", opts[:actor])
-        if opts[:actor] do
-          Map.put(data, "actor", opts[:actor])
-        else
-          data
-        end
 
       Repo.insert(%Object{data: obj_data})
     end
@@ -1275,8 +1233,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
         %{"type" => "Emoji"} -> true
         _ -> false
       end)
-      |> Enum.reduce(%{}, fn %{"icon" => %{"url" => url}, "name" => name}, acc ->
+      |> Map.new(fn %{"icon" => %{"url" => url}, "name" => name} ->
-        Map.put(acc, String.trim(name, ":"), url)
+        {String.trim(name, ":"), url}
       end)
 
     locked = data["manuallyApprovesFollowers"] || false
@@ -1322,18 +1280,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     }
 
     # nickname can be nil because of virtual actors
-    user_data =
+    if data["preferredUsername"] do
-      if data["preferredUsername"] do
+      Map.put(
-        Map.put(
+        user_data,
-          user_data,
+        :nickname,
-          :nickname,
+        "#{data["preferredUsername"]}@#{URI.parse(data["id"]).host}"
-          "#{data["preferredUsername"]}@#{URI.parse(data["id"]).host}"
+      )
-        )
+    else
-      else
+      Map.put(user_data, :nickname, nil)
-        Map.put(user_data, :nickname, nil)
+    end
-      end
-
-    {:ok, user_data}
   end
 
   def fetch_follow_information_for_user(user) do
@@ -1408,9 +1363,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp collection_private(_data), do: {:ok, true}
 
   def user_data_from_user_object(data) do
-    with {:ok, data} <- MRF.filter(data),
+    with {:ok, data} <- MRF.filter(data) do
-         {:ok, data} <- object_to_user_data(data) do
+      {:ok, object_to_user_data(data)}
-      {:ok, data}
     else
       e -> {:error, e}
     end
@@ -1418,15 +1372,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   def fetch_and_prepare_user_from_ap_id(ap_id) do
     with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id),
-         {:ok, data} <- user_data_from_user_object(data),
+         {:ok, data} <- user_data_from_user_object(data) do
-         data <- maybe_update_follow_information(data) do
+      {:ok, maybe_update_follow_information(data)}
-      {:ok, data}
     else
-      {:error, "Object has been deleted"} = e ->
+      {:error, "Object has been deleted" = e} ->
         Logger.debug("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
         {:error, e}
 
-      e ->
+      {:error, e} ->
         Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
         {:error, e}
     end
@@ -1449,8 +1402,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           |> Repo.insert()
           |> User.set_cache()
         end
-      else
-        e -> {:error, e}
       end
     end
   end
@@ -1464,7 +1415,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   # filter out broken threads
-  def contain_broken_threads(%Activity{} = activity, %User{} = user) do
+  defp contain_broken_threads(%Activity{} = activity, %User{} = user) do
     entire_thread_visible_for_user?(activity, user)
   end
 
@@ -1475,7 +1426,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   def fetch_direct_messages_query do
     Activity
-    |> restrict_type(%{"type" => "Create"})
+    |> restrict_type(%{type: "Create"})
     |> restrict_visibility(%{visibility: "direct"})
     |> order_by([activity], asc: activity.id)
   end

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -21,6 +21,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   alias Pleroma.Web.ActivityPub.UserView
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
+  alias Pleroma.Web.ControllerHelper
+  alias Pleroma.Web.Endpoint
   alias Pleroma.Web.FederatingPlug
   alias Pleroma.Web.Federator
 
@@ -75,8 +77,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     end
   end
 
-  def object(conn, %{"uuid" => uuid}) do
+  def object(conn, _) do
-    with ap_id <- o_status_url(conn, :object, uuid),
+    with ap_id <- Endpoint.url() <> conn.request_path,
          %Object{} = object <- Object.get_cached_by_ap_id(ap_id),
          {_, true} <- {:public?, Visibility.is_public?(object)} do
       conn
@@ -101,8 +103,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     conn
   end
 
-  def activity(conn, %{"uuid" => uuid}) do
+  def activity(conn, _params) do
-    with ap_id <- o_status_url(conn, :activity, uuid),
+    with ap_id <- Endpoint.url() <> conn.request_path,
          %Activity{} = activity <- Activity.normalize(ap_id),
          {_, true} <- {:public?, Visibility.is_public?(activity)} do
       conn
@@ -229,27 +231,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
       when page? in [true, "true"] do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
          {:ok, user} <- User.ensure_keys_present(user) do
-      activities =
+      # "include_poll_votes" is a hack because postgres generates inefficient
-        if params["max_id"] do
+      # queries when filtering by 'Answer', poll votes will be hidden by the
-          ActivityPub.fetch_user_activities(user, for_user, %{
+      # visibility filter in this case anyway
-            "max_id" => params["max_id"],
+      params =
-            # This is a hack because postgres generates inefficient queries when filtering by
+        params
-            # 'Answer', poll votes will be hidden by the visibility filter in this case anyway
+        |> Map.drop(["nickname", "page"])
-            "include_poll_votes" => true,
+        |> Map.put("include_poll_votes", true)
-            "limit" => 10
+        |> Map.new(fn {k, v} -> {String.to_existing_atom(k), v} end)
-          })
+
-        else
+      activities = ActivityPub.fetch_user_activities(user, for_user, params)
-          ActivityPub.fetch_user_activities(user, for_user, %{
-            "limit" => 10,
-            "include_poll_votes" => true
-          })
-        end
 
       conn
       |> put_resp_content_type("application/activity+json")
       |> put_view(UserView)
       |> render("activity_collection_page.json", %{
         activities: activities,
+        pagination: ControllerHelper.get_pagination_fields(conn, activities),
         iri: "#{user.ap_id}/outbox"
       })
     end
@@ -352,21 +350,24 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
         %{"nickname" => nickname, "page" => page?} = params
       )
       when page? in [true, "true"] do
+    params =
+      params
+      |> Map.drop(["nickname", "page"])
+      |> Map.put("blocking_user", user)
+      |> Map.put("user", user)
+      |> Map.new(fn {k, v} -> {String.to_existing_atom(k), v} end)
+
     activities =
-      if params["max_id"] do
+      [user.ap_id | User.following(user)]
-        ActivityPub.fetch_activities([user.ap_id | User.following(user)], %{
+      |> ActivityPub.fetch_activities(params)
-          "max_id" => params["max_id"],
+      |> Enum.reverse()
-          "limit" => 10
-        })
-      else
-        ActivityPub.fetch_activities([user.ap_id | User.following(user)], %{"limit" => 10})
-      end
 
     conn
     |> put_resp_content_type("application/activity+json")
     |> put_view(UserView)
     |> render("activity_collection_page.json", %{
       activities: activities,
+      pagination: ControllerHelper.get_pagination_fields(conn, activities),
       iri: "#{user.ap_id}/inbox"
     })
   end

lib/pleroma/web/activity_pub/builder.ex

@@ -7,9 +7,12 @@ defmodule Pleroma.Web.ActivityPub.Builder do
 
   alias Pleroma.Object
   alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
 
+  require Pleroma.Constants
+
   @spec emoji_react(User.t(), Object.t(), String.t()) :: {:ok, map(), keyword()}
   def emoji_react(actor, object, emoji) do
     with {:ok, data, meta} <- object_action(actor, object) do
@@ -83,6 +86,34 @@ defmodule Pleroma.Web.ActivityPub.Builder do
     end
   end
 
+  @spec announce(User.t(), Object.t(), keyword()) :: {:ok, map(), keyword()}
+  def announce(actor, object, options \\ []) do
+    public? = Keyword.get(options, :public, false)
+
+    to =
+      cond do
+        actor.ap_id == Relay.relay_ap_id() ->
+          [actor.follower_address]
+
+        public? ->
+          [actor.follower_address, object.data["actor"], Pleroma.Constants.as_public()]
+
+        true ->
+          [actor.follower_address, object.data["actor"]]
+      end
+
+    {:ok,
+     %{
+       "id" => Utils.generate_activity_id(),
+       "actor" => actor.ap_id,
+       "object" => object.data["id"],
+       "to" => to,
+       "context" => object.data["context"],
+       "type" => "Announce",
+       "published" => Utils.make_date()
+     }, []}
+  end
+
   @spec object_action(User.t(), Object.t()) :: {:ok, map(), keyword()}
   defp object_action(actor, object) do
     object_actor = User.get_cached_by_ap_id(object.data["actor"])

lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex

@@ -0,0 +1,97 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do
+  require Logger
+
+  alias Pleroma.Config
+
+  @moduledoc "Detect new emojis by their shortcode and steals them"
+  @behaviour Pleroma.Web.ActivityPub.MRF
+
+  defp remote_host?(host), do: host != Config.get([Pleroma.Web.Endpoint, :url, :host])
+
+  defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], [])
+
+  defp steal_emoji({shortcode, url}) do
+    url = Pleroma.Web.MediaProxy.url(url)
+    {:ok, response} = Pleroma.HTTP.get(url)
+    size_limit = Config.get([:mrf_steal_emoji, :size_limit], 50_000)
+
+    if byte_size(response.body) <= size_limit do
+      emoji_dir_path =
+        Config.get(
+          [:mrf_steal_emoji, :path],
+          Path.join(Config.get([:instance, :static_dir]), "emoji/stolen")
+        )
+
+      extension =
+        url
+        |> URI.parse()
+        |> Map.get(:path)
+        |> Path.basename()
+        |> Path.extname()
+
+      file_path = Path.join([emoji_dir_path, shortcode <> (extension || ".png")])
+
+      try do
+        :ok = File.write(file_path, response.body)
+
+        shortcode
+      rescue
+        e ->
+          Logger.warn("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}")
+          nil
+      end
+    else
+      Logger.debug(
+        "MRF.StealEmojiPolicy: :#{shortcode}: at #{url} (#{byte_size(response.body)} B) over size limit (#{
+          size_limit
+        } B)"
+      )
+
+      nil
+    end
+  rescue
+    e ->
+      Logger.warn("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}")
+      nil
+  end
+
+  @impl true
+  def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = message) do
+    host = URI.parse(actor).host
+
+    if remote_host?(host) and accept_host?(host) do
+      installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
+
+      new_emojis =
+        foreign_emojis
+        |> Enum.filter(fn {shortcode, _url} -> shortcode not in installed_emoji end)
+        |> Enum.filter(fn {shortcode, _url} ->
+          reject_emoji? =
+            Config.get([:mrf_steal_emoji, :rejected_shortcodes], [])
+            |> Enum.find(false, fn regex -> String.match?(shortcode, regex) end)
+
+          !reject_emoji?
+        end)
+        |> Enum.map(&steal_emoji(&1))
+        |> Enum.filter(& &1)
+
+      if !Enum.empty?(new_emojis) do
+        Logger.info("Stole new emojis: #{inspect(new_emojis)}")
+        Pleroma.Emoji.reload()
+      end
+    end
+
+    {:ok, message}
+  end
+
+  def filter(message), do: {:ok, message}
+
+  @impl true
+  def describe do
+    {:ok, %{}}
+  end
+end

lib/pleroma/web/activity_pub/object_validator.ex

@@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
 
   alias Pleroma.Object
   alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
@@ -58,6 +59,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     end
   end
 
+  def validate(%{"type" => "Announce"} = object, meta) do
+    with {:ok, object} <-
+           object
+           |> AnnounceValidator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
+      object = stringify_keys(object |> Map.from_struct())
+      {:ok, object, meta}
+    end
+  end
+
   def stringify_keys(%{__struct__: _} = object) do
     object
     |> Map.from_struct()
@@ -77,7 +88,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
 
   def fetch_actor_and_object(object) do
     fetch_actor(object)
-    Object.normalize(object["object"])
+    Object.normalize(object["object"], true)
     :ok
   end
 end

lib/pleroma/web/activity_pub/object_validators/announce_validator.ex

@@ -0,0 +1,101 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator do
+  use Ecto.Schema
+
+  alias Pleroma.Object
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
+  alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.Web.ActivityPub.Visibility
+
+  import Ecto.Changeset
+  import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+
+  require Pleroma.Constants
+
+  @primary_key false
+
+  embedded_schema do
+    field(:id, Types.ObjectID, primary_key: true)
+    field(:type, :string)
+    field(:object, Types.ObjectID)
+    field(:actor, Types.ObjectID)
+    field(:context, :string, autogenerate: {Utils, :generate_context_id, []})
+    field(:to, Types.Recipients, default: [])
+    field(:cc, Types.Recipients, default: [])
+    field(:published, Types.DateTime)
+  end
+
+  def cast_and_validate(data) do
+    data
+    |> cast_data()
+    |> validate_data()
+  end
+
+  def cast_data(data) do
+    %__MODULE__{}
+    |> changeset(data)
+  end
+
+  def changeset(struct, data) do
+    struct
+    |> cast(data, __schema__(:fields))
+    |> fix_after_cast()
+  end
+
+  def fix_after_cast(cng) do
+    cng
+  end
+
+  def validate_data(data_cng) do
+    data_cng
+    |> validate_inclusion(:type, ["Announce"])
+    |> validate_required([:id, :type, :object, :actor, :to, :cc])
+    |> validate_actor_presence()
+    |> validate_object_presence()
+    |> validate_existing_announce()
+    |> validate_announcable()
+  end
+
+  def validate_announcable(cng) do
+    with actor when is_binary(actor) <- get_field(cng, :actor),
+         object when is_binary(object) <- get_field(cng, :object),
+         %User{} = actor <- User.get_cached_by_ap_id(actor),
+         %Object{} = object <- Object.get_cached_by_ap_id(object),
+         false <- Visibility.is_public?(object) do
+      same_actor = object.data["actor"] == actor.ap_id
+      is_public = Pleroma.Constants.as_public() in (get_field(cng, :to) ++ get_field(cng, :cc))
+
+      cond do
+        same_actor && is_public ->
+          cng
+          |> add_error(:actor, "can not announce this object publicly")
+
+        !same_actor ->
+          cng
+          |> add_error(:actor, "can not announce this object")
+
+        true ->
+          cng
+      end
+    else
+      _ -> cng
+    end
+  end
+
+  def validate_existing_announce(cng) do
+    actor = get_field(cng, :actor)
+    object = get_field(cng, :object)
+
+    if actor && object && Utils.get_existing_announce(actor, %{data: %{"id" => object}}) do
+      cng
+      |> add_error(:actor, "already announced this object")
+      |> add_error(:object, "already announced by this actor")
+    else
+      cng
+    end
+  end
+end

lib/pleroma/web/activity_pub/pipeline.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.ActivityPub.Pipeline do
   alias Pleroma.Activity
+  alias Pleroma.Config
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -44,7 +45,7 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do
 
   defp maybe_federate(%Activity{} = activity, meta) do
     with {:ok, local} <- Keyword.fetch(meta, :local) do
-      do_not_federate = meta[:do_not_federate]
+      do_not_federate = meta[:do_not_federate] || !Config.get([:instance, :federating])
 
       if !do_not_federate && local do
         Federator.publish(activity)

lib/pleroma/web/activity_pub/relay.ex

@@ -4,9 +4,10 @@
 
 defmodule Pleroma.Web.ActivityPub.Relay do
   alias Pleroma.Activity
-  alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.Visibility
+  alias Pleroma.Web.CommonAPI
   require Logger
 
   @relay_nickname "relay"
@@ -48,11 +49,11 @@ defmodule Pleroma.Web.ActivityPub.Relay do
     end
   end
 
-  @spec publish(any()) :: {:ok, Activity.t(), Object.t()} | {:error, any()}
+  @spec publish(any()) :: {:ok, Activity.t()} | {:error, any()}
   def publish(%Activity{data: %{"type" => "Create"}} = activity) do
     with %User{} = user <- get_actor(),
-         %Object{} = object <- Object.normalize(activity) do
+         true <- Visibility.is_public?(activity) do
-      ActivityPub.announce(user, object, nil, true, false)
+      CommonAPI.repeat(activity.id, user)
     else
       error -> format_error(error)
     end

lib/pleroma/web/activity_pub/side_effects.ex

@@ -27,6 +27,24 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
     {:ok, object, meta}
   end
 
+  # Tasks this handles:
+  # - Add announce to object
+  # - Set up notification
+  # - Stream out the announce
+  def handle(%{data: %{"type" => "Announce"}} = object, meta) do
+    announced_object = Object.get_by_ap_id(object.data["object"])
+    user = User.get_cached_by_ap_id(object.data["actor"])
+
+    Utils.add_announce_to_object(object, announced_object)
+
+    if !User.is_internal_user?(user) do
+      Notification.create_notifications(object)
+      ActivityPub.stream_out(object)
+    end
+
+    {:ok, object, meta}
+  end
+
   def handle(%{data: %{"type" => "Undo", "object" => undone_object}} = object, meta) do
     with undone_object <- Activity.get_by_ap_id(undone_object),
          :ok <- handle_undoing(undone_object) do

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   alias Pleroma.Activity
   alias Pleroma.EarmarkRenderer
   alias Pleroma.FollowingRelationship
+  alias Pleroma.Maps
   alias Pleroma.Object
   alias Pleroma.Object.Containment
   alias Pleroma.Repo
@@ -208,12 +209,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> Map.put("conversation", context)
   end
 
-  defp add_if_present(map, _key, nil), do: map
-
-  defp add_if_present(map, key, value) do
-    Map.put(map, key, value)
-  end
-
   def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachment) do
     attachments =
       Enum.map(attachment, fn data ->
@@ -241,13 +236,13 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
         attachment_url =
           %{"href" => href}
-          |> add_if_present("mediaType", media_type)
+          |> Maps.put_if_present("mediaType", media_type)
-          |> add_if_present("type", Map.get(url || %{}, "type"))
+          |> Maps.put_if_present("type", Map.get(url || %{}, "type"))
 
         %{"url" => [attachment_url]}
-        |> add_if_present("mediaType", media_type)
+        |> Maps.put_if_present("mediaType", media_type)
-        |> add_if_present("type", data["type"])
+        |> Maps.put_if_present("type", data["type"])
-        |> add_if_present("name", data["name"])
+        |> Maps.put_if_present("name", data["name"])
       end)
 
     Map.put(object, "attachment", attachments)
@@ -662,7 +657,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> handle_incoming(options)
   end
 
-  def handle_incoming(%{"type" => type} = data, _options) when type in ["Like", "EmojiReact"] do
+  def handle_incoming(%{"type" => type} = data, _options)
+      when type in ["Like", "EmojiReact", "Announce"] do
     with :ok <- ObjectValidator.fetch_actor_and_object(data),
          {:ok, activity, _meta} <-
            Pipeline.common_pipeline(data, local: false) do
@@ -672,21 +668,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     end
   end
 
-  def handle_incoming(
-        %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data,
-        _options
-      ) do
-    with actor <- Containment.get_actor(data),
-         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
-         {:ok, object} <- get_embedded_obj_helper(object_id, actor),
-         public <- Visibility.is_public?(data),
-         {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false, public) do
-      {:ok, activity}
-    else
-      _e -> :error
-    end
-  end
-
   def handle_incoming(
         %{"type" => "Update", "object" => %{"type" => object_type} = object, "actor" => actor_id} =
           data,
@@ -1059,10 +1040,14 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     Map.put(object, "tag", tags)
   end
 
+  # TODO These should be added on our side on insertion, it doesn't make much
+  # sense to regenerate these all the time
   def add_mention_tags(object) do
-    {enabled_receivers, disabled_receivers} = Utils.get_notified_from_object(object)
+    to = object["to"] || []
-    potential_receivers = enabled_receivers ++ disabled_receivers
+    cc = object["cc"] || []
-    mentions = Enum.map(potential_receivers, &build_mention_tag/1)
+    mentioned = User.get_users_from_set(to ++ cc, local_only: false)
+
+    mentions = Enum.map(mentioned, &build_mention_tag/1)
 
     tags = object["tag"] || []
     Map.put(object, "tag", tags ++ mentions)

lib/pleroma/web/activity_pub/utils.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   alias Ecto.UUID
   alias Pleroma.Activity
   alias Pleroma.Config
+  alias Pleroma.Maps
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo
@@ -244,7 +245,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   Inserts a full object if it is contained in an activity.
   """
   def insert_full_object(%{"object" => %{"type" => type} = object_data} = map)
-      when is_map(object_data) and type in @supported_object_types do
+      when type in @supported_object_types do
     with {:ok, object} <- Object.create(object_data) do
       map = Map.put(map, "object", object.data["id"])
 
@@ -307,7 +308,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
       "cc" => cc,
       "context" => object.data["context"]
     }
-    |> maybe_put("id", activity_id)
+    |> Maps.put_if_present("id", activity_id)
   end
 
   def make_emoji_reaction_data(user, object, emoji, activity_id) do
@@ -477,7 +478,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
       "object" => followed_id,
       "state" => "pending"
     }
-    |> maybe_put("id", activity_id)
+    |> Maps.put_if_present("id", activity_id)
   end
 
   def fetch_latest_follow(%User{ap_id: follower_id}, %User{ap_id: followed_id}) do
@@ -546,7 +547,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
       "cc" => [],
       "context" => object.data["context"]
     }
-    |> maybe_put("id", activity_id)
+    |> Maps.put_if_present("id", activity_id)
   end
 
   def make_announce_data(
@@ -563,7 +564,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
       "cc" => [Pleroma.Constants.as_public()],
       "context" => object.data["context"]
     }
-    |> maybe_put("id", activity_id)
+    |> Maps.put_if_present("id", activity_id)
   end
 
   def make_undo_data(
@@ -582,7 +583,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
       "cc" => [Pleroma.Constants.as_public()],
       "context" => context
     }
-    |> maybe_put("id", activity_id)
+    |> Maps.put_if_present("id", activity_id)
   end
 
   @spec add_announce_to_object(Activity.t(), Object.t()) ::
@@ -627,7 +628,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
       "to" => [followed.ap_id],
       "object" => follow_activity.data
     }
-    |> maybe_put("id", activity_id)
+    |> Maps.put_if_present("id", activity_id)
   end
 
   #### Block-related helpers
@@ -650,7 +651,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
       "to" => [blocked.ap_id],
       "object" => blocked.ap_id
     }
-    |> maybe_put("id", activity_id)
+    |> Maps.put_if_present("id", activity_id)
   end
 
   #### Create-related helpers
@@ -740,12 +741,12 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   def get_reports(params, page, page_size) do
     params =
       params
-      |> Map.put("type", "Flag")
+      |> Map.put(:type, "Flag")
-      |> Map.put("skip_preload", true)
+      |> Map.put(:skip_preload, true)
-      |> Map.put("preload_report_notes", true)
+      |> Map.put(:preload_report_notes, true)
-      |> Map.put("total", true)
+      |> Map.put(:total, true)
-      |> Map.put("limit", page_size)
+      |> Map.put(:limit, page_size)
-      |> Map.put("offset", (page - 1) * page_size)
+      |> Map.put(:offset, (page - 1) * page_size)
 
     ActivityPub.fetch_activities([], params, :offset)
   end
@@ -870,7 +871,4 @@ defmodule Pleroma.Web.ActivityPub.Utils do
     |> where([a, object: o], fragment("(?)->>'type' = 'Answer'", o.data))
     |> Repo.all()
   end
-
-  def maybe_put(map, _key, nil), do: map
-  def maybe_put(map, key, value), do: Map.put(map, key, value)
 end

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -213,34 +213,24 @@ defmodule Pleroma.Web.ActivityPub.UserView do
     |> Map.merge(Utils.make_json_ld_header())
   end
 
-  def render("activity_collection_page.json", %{activities: activities, iri: iri}) do
+  def render("activity_collection_page.json", %{
-    # this is sorted chronologically, so first activity is the newest (max)
+        activities: activities,
-    {max_id, min_id, collection} =
+        iri: iri,
-      if length(activities) > 0 do
+        pagination: pagination
-        {
+      }) do
-          Enum.at(activities, 0).id,
+    collection =
-          Enum.at(Enum.reverse(activities), 0).id,
+      Enum.map(activities, fn activity ->
-          Enum.map(activities, fn act ->
+        {:ok, data} = Transmogrifier.prepare_outgoing(activity.data)
-            {:ok, data} = Transmogrifier.prepare_outgoing(act.data)
+        data
-            data
+      end)
-          end)
-        }
-      else
-        {
-          0,
-          0,
-          []
-        }
-      end
 
     %{
-      "id" => "#{iri}?max_id=#{max_id}&page=true",
       "type" => "OrderedCollectionPage",
       "partOf" => iri,
-      "orderedItems" => collection,
+      "orderedItems" => collection
-      "next" => "#{iri}?max_id=#{min_id}&page=true"
     }
     |> Map.merge(Utils.make_json_ld_header())
+    |> Map.merge(pagination)
   end
 
   defp maybe_put_total_items(map, false, _total), do: map

lib/pleroma/web/admin_api/controllers/admin_api_controller.ex

@@ -7,38 +7,24 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
 
   import Pleroma.Web.ControllerHelper, only: [json_response: 3]
 
-  alias Pleroma.Activity
   alias Pleroma.Config
-  alias Pleroma.ConfigDB
   alias Pleroma.MFA
   alias Pleroma.ModerationLog
   alias Pleroma.Plugs.OAuthScopesPlug
-  alias Pleroma.ReportNote
   alias Pleroma.Stats
   alias Pleroma.User
-  alias Pleroma.UserInviteToken
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Builder
   alias Pleroma.Web.ActivityPub.Pipeline
-  alias Pleroma.Web.ActivityPub.Relay
-  alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.AdminAPI
   alias Pleroma.Web.AdminAPI.AccountView
-  alias Pleroma.Web.AdminAPI.ConfigView
   alias Pleroma.Web.AdminAPI.ModerationLogView
-  alias Pleroma.Web.AdminAPI.Report
-  alias Pleroma.Web.AdminAPI.ReportView
   alias Pleroma.Web.AdminAPI.Search
-  alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.Endpoint
-  alias Pleroma.Web.MastodonAPI
-  alias Pleroma.Web.MastodonAPI.AppView
-  alias Pleroma.Web.OAuth.App
   alias Pleroma.Web.Router
 
   require Logger
 
-  @descriptions Pleroma.Docs.JSON.compile()
   @users_page_size 50
 
   plug(
@@ -69,53 +55,24 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
          ]
   )
 
-  plug(OAuthScopesPlug, %{scopes: ["read:invites"], admin: true} when action == :invites)
-
-  plug(
-    OAuthScopesPlug,
-    %{scopes: ["write:invites"], admin: true}
-    when action in [:create_invite_token, :revoke_invite, :email_invite]
-  )
-
   plug(
     OAuthScopesPlug,
     %{scopes: ["write:follows"], admin: true}
-    when action in [:user_follow, :user_unfollow, :relay_follow, :relay_unfollow]
+    when action in [:user_follow, :user_unfollow]
-  )
-
-  plug(
-    OAuthScopesPlug,
-    %{scopes: ["read:reports"], admin: true}
-    when action in [:list_reports, :report_show]
-  )
-
-  plug(
-    OAuthScopesPlug,
-    %{scopes: ["write:reports"], admin: true}
-    when action in [:reports_update, :report_notes_create, :report_notes_delete]
   )
 
   plug(
     OAuthScopesPlug,
     %{scopes: ["read:statuses"], admin: true}
-    when action in [:list_statuses, :list_user_statuses, :list_instance_statuses, :status_show]
+    when action in [:list_user_statuses, :list_instance_statuses]
-  )
-
-  plug(
-    OAuthScopesPlug,
-    %{scopes: ["write:statuses"], admin: true}
-    when action in [:status_update, :status_delete]
   )
 
   plug(
     OAuthScopesPlug,
     %{scopes: ["read"], admin: true}
     when action in [
-           :config_show,
            :list_log,
            :stats,
-           :relay_list,
-           :config_descriptions,
            :need_reboot
          ]
   )
@@ -125,18 +82,13 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     %{scopes: ["write"], admin: true}
     when action in [
            :restart,
-           :config_update,
            :resend_confirmation_email,
            :confirm_email,
-           :oauth_app_create,
-           :oauth_app_list,
-           :oauth_app_update,
-           :oauth_app_delete,
            :reload_emoji
          ]
   )
 
-  action_fallback(:errors)
+  action_fallback(AdminAPI.FallbackController)
 
   def user_delete(conn, %{"nickname" => nickname}) do
     user_delete(conn, %{"nicknames" => [nickname]})
@@ -274,10 +226,10 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
 
     activities =
       ActivityPub.fetch_statuses(nil, %{
-        "instance" => instance,
+        instance: instance,
-        "limit" => page_size,
+        limit: page_size,
-        "offset" => (page - 1) * page_size,
+        offset: (page - 1) * page_size,
-        "exclude_reblogs" => !with_reblogs && "true"
+        exclude_reblogs: not with_reblogs
       })
 
     conn
@@ -294,13 +246,13 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
 
       activities =
         ActivityPub.fetch_user_activities(user, nil, %{
-          "limit" => page_size,
+          limit: page_size,
-          "godmode" => godmode,
+          godmode: godmode,
-          "exclude_reblogs" => !with_reblogs && "true"
+          exclude_reblogs: not with_reblogs
         })
 
       conn
-      |> put_view(MastodonAPI.StatusView)
+      |> put_view(AdminAPI.StatusView)
       |> render("index.json", %{activities: activities, as: :activity})
     else
       _ -> {:error, :not_found}
@@ -537,119 +489,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     render_error(conn, :forbidden, "You can't revoke your own admin status.")
   end
 
-  def relay_list(conn, _params) do
-    with {:ok, list} <- Relay.list() do
-      json(conn, %{relays: list})
-    else
-      _ ->
-        conn
-        |> put_status(500)
-    end
-  end
-
-  def relay_follow(%{assigns: %{user: admin}} = conn, %{"relay_url" => target}) do
-    with {:ok, _message} <- Relay.follow(target) do
-      ModerationLog.insert_log(%{
-        action: "relay_follow",
-        actor: admin,
-        target: target
-      })
-
-      json(conn, target)
-    else
-      _ ->
-        conn
-        |> put_status(500)
-        |> json(target)
-    end
-  end
-
-  def relay_unfollow(%{assigns: %{user: admin}} = conn, %{"relay_url" => target}) do
-    with {:ok, _message} <- Relay.unfollow(target) do
-      ModerationLog.insert_log(%{
-        action: "relay_unfollow",
-        actor: admin,
-        target: target
-      })
-
-      json(conn, target)
-    else
-      _ ->
-        conn
-        |> put_status(500)
-        |> json(target)
-    end
-  end
-
-  @doc "Sends registration invite via email"
-  def email_invite(%{assigns: %{user: user}} = conn, %{"email" => email} = params) do
-    with {_, false} <- {:registrations_open, Config.get([:instance, :registrations_open])},
-         {_, true} <- {:invites_enabled, Config.get([:instance, :invites_enabled])},
-         {:ok, invite_token} <- UserInviteToken.create_invite(),
-         email <-
-           Pleroma.Emails.UserEmail.user_invitation_email(
-             user,
-             invite_token,
-             email,
-             params["name"]
-           ),
-         {:ok, _} <- Pleroma.Emails.Mailer.deliver(email) do
-      json_response(conn, :no_content, "")
-    else
-      {:registrations_open, _} ->
-        errors(
-          conn,
-          {:error, "To send invites you need to set the `registrations_open` option to false."}
-        )
-
-      {:invites_enabled, _} ->
-        errors(
-          conn,
-          {:error, "To send invites you need to set the `invites_enabled` option to true."}
-        )
-    end
-  end
-
-  @doc "Create an account registration invite token"
-  def create_invite_token(conn, params) do
-    opts = %{}
-
-    opts =
-      if params["max_use"],
-        do: Map.put(opts, :max_use, params["max_use"]),
-        else: opts
-
-    opts =
-      if params["expires_at"],
-        do: Map.put(opts, :expires_at, params["expires_at"]),
-        else: opts
-
-    {:ok, invite} = UserInviteToken.create_invite(opts)
-
-    json(conn, AccountView.render("invite.json", %{invite: invite}))
-  end
-
-  @doc "Get list of created invites"
-  def invites(conn, _params) do
-    invites = UserInviteToken.list_invites()
-
-    conn
-    |> put_view(AccountView)
-    |> render("invites.json", %{invites: invites})
-  end
-
-  @doc "Revokes invite by token"
-  def revoke_invite(conn, %{"token" => token}) do
-    with {:ok, invite} <- UserInviteToken.find_by_token(token),
-         {:ok, updated_invite} = UserInviteToken.update_invite(invite, %{used: true}) do
-      conn
-      |> put_view(AccountView)
-      |> render("invite.json", %{invite: updated_invite})
-    else
-      nil -> {:error, :not_found}
-    end
-  end
-
   @doc "Get a password reset token (base64 string) for given nickname"
   def get_password_reset(conn, %{"nickname" => nickname}) do
     (%User{local: true} = user) = User.get_cached_by_nickname(nickname)
@@ -705,7 +544,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
         %{assigns: %{user: admin}} = conn,
         %{"nickname" => nickname} = params
       ) do
-    with {_, user} <- {:user, User.get_cached_by_nickname(nickname)},
+    with {_, %User{} = user} <- {:user, User.get_cached_by_nickname(nickname)},
          {:ok, _user} <-
            User.update_as_admin(user, params) do
       ModerationLog.insert_log(%{
@@ -727,155 +566,12 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
       json(conn, %{status: "success"})
     else
       {:error, changeset} ->
-        {_, {error, _}} = Enum.at(changeset.errors, 0)
+        errors = Map.new(changeset.errors, fn {key, {error, _}} -> {key, error} end)
-        json(conn, %{error: "New password #{error}."})
+
+        json(conn, %{errors: errors})
 
       _ ->
-        json(conn, %{error: "Unable to change password."})
+        json(conn, %{error: "Unable to update user."})
-    end
-  end
-
-  def list_reports(conn, params) do
-    {page, page_size} = page_params(params)
-
-    reports = Utils.get_reports(params, page, page_size)
-
-    conn
-    |> put_view(ReportView)
-    |> render("index.json", %{reports: reports})
-  end
-
-  def report_show(conn, %{"id" => id}) do
-    with %Activity{} = report <- Activity.get_by_id(id) do
-      conn
-      |> put_view(ReportView)
-      |> render("show.json", Report.extract_report_info(report))
-    else
-      _ -> {:error, :not_found}
-    end
-  end
-
-  def reports_update(%{assigns: %{user: admin}} = conn, %{"reports" => reports}) do
-    result =
-      reports
-      |> Enum.map(fn report ->
-        with {:ok, activity} <- CommonAPI.update_report_state(report["id"], report["state"]) do
-          ModerationLog.insert_log(%{
-            action: "report_update",
-            actor: admin,
-            subject: activity
-          })
-
-          activity
-        else
-          {:error, message} -> %{id: report["id"], error: message}
-        end
-      end)
-
-    case Enum.any?(result, &Map.has_key?(&1, :error)) do
-      true -> json_response(conn, :bad_request, result)
-      false -> json_response(conn, :no_content, "")
-    end
-  end
-
-  def report_notes_create(%{assigns: %{user: user}} = conn, %{
-        "id" => report_id,
-        "content" => content
-      }) do
-    with {:ok, _} <- ReportNote.create(user.id, report_id, content) do
-      ModerationLog.insert_log(%{
-        action: "report_note",
-        actor: user,
-        subject: Activity.get_by_id(report_id),
-        text: content
-      })
-
-      json_response(conn, :no_content, "")
-    else
-      _ -> json_response(conn, :bad_request, "")
-    end
-  end
-
-  def report_notes_delete(%{assigns: %{user: user}} = conn, %{
-        "id" => note_id,
-        "report_id" => report_id
-      }) do
-    with {:ok, note} <- ReportNote.destroy(note_id) do
-      ModerationLog.insert_log(%{
-        action: "report_note_delete",
-        actor: user,
-        subject: Activity.get_by_id(report_id),
-        text: note.content
-      })
-
-      json_response(conn, :no_content, "")
-    else
-      _ -> json_response(conn, :bad_request, "")
-    end
-  end
-
-  def list_statuses(%{assigns: %{user: _admin}} = conn, params) do
-    godmode = params["godmode"] == "true" || params["godmode"] == true
-    local_only = params["local_only"] == "true" || params["local_only"] == true
-    with_reblogs = params["with_reblogs"] == "true" || params["with_reblogs"] == true
-    {page, page_size} = page_params(params)
-
-    activities =
-      ActivityPub.fetch_statuses(nil, %{
-        "godmode" => godmode,
-        "local_only" => local_only,
-        "limit" => page_size,
-        "offset" => (page - 1) * page_size,
-        "exclude_reblogs" => !with_reblogs && "true"
-      })
-
-    conn
-    |> put_view(AdminAPI.StatusView)
-    |> render("index.json", %{activities: activities, as: :activity})
-  end
-
-  def status_show(conn, %{"id" => id}) do
-    with %Activity{} = activity <- Activity.get_by_id(id) do
-      conn
-      |> put_view(MastodonAPI.StatusView)
-      |> render("show.json", %{activity: activity})
-    else
-      _ -> errors(conn, {:error, :not_found})
-    end
-  end
-
-  def status_update(%{assigns: %{user: admin}} = conn, %{"id" => id} = params) do
-    params =
-      params
-      |> Map.take(["sensitive", "visibility"])
-      |> Map.new(fn {key, value} -> {String.to_existing_atom(key), value} end)
-
-    with {:ok, activity} <- CommonAPI.update_activity_scope(id, params) do
-      {:ok, sensitive} = Ecto.Type.cast(:boolean, params[:sensitive])
-
-      ModerationLog.insert_log(%{
-        action: "status_update",
-        actor: admin,
-        subject: activity,
-        sensitive: sensitive,
-        visibility: params[:visibility]
-      })
-
-      conn
-      |> put_view(MastodonAPI.StatusView)
-      |> render("show.json", %{activity: activity})
-    end
-  end
-
-  def status_delete(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {:ok, %Activity{}} <- CommonAPI.delete(id, user) do
-      ModerationLog.insert_log(%{
-        action: "status_delete",
-        actor: user,
-        subject_id: id
-      })
-
-      json(conn, %{})
     end
   end
 
@@ -897,107 +593,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     |> render("index.json", %{log: log})
   end
 
-  def config_descriptions(conn, _params) do
-    descriptions = Enum.filter(@descriptions, &whitelisted_config?/1)
-
-    json(conn, descriptions)
-  end
-
-  def config_show(conn, %{"only_db" => true}) do
-    with :ok <- configurable_from_database(conn) do
-      configs = Pleroma.Repo.all(ConfigDB)
-
-      conn
-      |> put_view(ConfigView)
-      |> render("index.json", %{configs: configs})
-    end
-  end
-
-  def config_show(conn, _params) do
-    with :ok <- configurable_from_database(conn) do
-      configs = ConfigDB.get_all_as_keyword()
-
-      merged =
-        Config.Holder.default_config()
-        |> ConfigDB.merge(configs)
-        |> Enum.map(fn {group, values} ->
-          Enum.map(values, fn {key, value} ->
-            db =
-              if configs[group][key] do
-                ConfigDB.get_db_keys(configs[group][key], key)
-              end
-
-            db_value = configs[group][key]
-
-            merged_value =
-              if !is_nil(db_value) and Keyword.keyword?(db_value) and
-                   ConfigDB.sub_key_full_update?(group, key, Keyword.keys(db_value)) do
-                ConfigDB.merge_group(group, key, value, db_value)
-              else
-                value
-              end
-
-            setting = %{
-              group: ConfigDB.convert(group),
-              key: ConfigDB.convert(key),
-              value: ConfigDB.convert(merged_value)
-            }
-
-            if db, do: Map.put(setting, :db, db), else: setting
-          end)
-        end)
-        |> List.flatten()
-
-      json(conn, %{configs: merged, need_reboot: Restarter.Pleroma.need_reboot?()})
-    end
-  end
-
-  def config_update(conn, %{"configs" => configs}) do
-    with :ok <- configurable_from_database(conn) do
-      {_errors, results} =
-        configs
-        |> Enum.filter(&whitelisted_config?/1)
-        |> Enum.map(fn
-          %{"group" => group, "key" => key, "delete" => true} = params ->
-            ConfigDB.delete(%{group: group, key: key, subkeys: params["subkeys"]})
-
-          %{"group" => group, "key" => key, "value" => value} ->
-            ConfigDB.update_or_create(%{group: group, key: key, value: value})
-        end)
-        |> Enum.split_with(fn result -> elem(result, 0) == :error end)
-
-      {deleted, updated} =
-        results
-        |> Enum.map(fn {:ok, config} ->
-          Map.put(config, :db, ConfigDB.get_db_keys(config))
-        end)
-        |> Enum.split_with(fn config ->
-          Ecto.get_meta(config, :state) == :deleted
-        end)
-
-      Config.TransferTask.load_and_update_env(deleted, false)
-
-      if !Restarter.Pleroma.need_reboot?() do
-        changed_reboot_settings? =
-          (updated ++ deleted)
-          |> Enum.any?(fn config ->
-            group = ConfigDB.from_string(config.group)
-            key = ConfigDB.from_string(config.key)
-            value = ConfigDB.from_binary(config.value)
-            Config.TransferTask.pleroma_need_restart?(group, key, value)
-          end)
-
-        if changed_reboot_settings?, do: Restarter.Pleroma.need_reboot()
-      end
-
-      conn
-      |> put_view(ConfigView)
-      |> render("index.json", %{configs: updated, need_reboot: Restarter.Pleroma.need_reboot?()})
-    end
-  end
-
   def restart(conn, _params) do
-    with :ok <- configurable_from_database(conn) do
+    with :ok <- configurable_from_database() do
       Restarter.Pleroma.restart(Config.get(:env), 50)
 
       json(conn, %{})
@@ -1008,39 +605,14 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     json(conn, %{need_reboot: Restarter.Pleroma.need_reboot?()})
   end
 
-  defp configurable_from_database(conn) do
+  defp configurable_from_database do
     if Config.get(:configurable_from_database) do
       :ok
     else
-      errors(
+      {:error, "To use this endpoint you need to enable configuration from database."}
-        conn,
-        {:error, "To use this endpoint you need to enable configuration from database."}
-      )
     end
   end
 
-  defp whitelisted_config?(group, key) do
-    if whitelisted_configs = Config.get(:database_config_whitelist) do
-      Enum.any?(whitelisted_configs, fn
-        {whitelisted_group} ->
-          group == inspect(whitelisted_group)
-
-        {whitelisted_group, whitelisted_key} ->
-          group == inspect(whitelisted_group) && key == inspect(whitelisted_key)
-      end)
-    else
-      true
-    end
-  end
-
-  defp whitelisted_config?(%{"group" => group, "key" => key}) do
-    whitelisted_config?(group, key)
-  end
-
-  defp whitelisted_config?(%{:group => group} = config) do
-    whitelisted_config?(group, config[:key])
-  end
-
   def reload_emoji(conn, _params) do
     Pleroma.Emoji.reload()
 
@@ -1075,113 +647,12 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     conn |> json("")
   end
 
-  def oauth_app_create(conn, params) do
-    params =
-      if params["name"] do
-        Map.put(params, "client_name", params["name"])
-      else
-        params
-      end
-
-    result =
-      case App.create(params) do
-        {:ok, app} ->
-          AppView.render("show.json", %{app: app, admin: true})
-
-        {:error, changeset} ->
-          App.errors(changeset)
-      end
-
-    json(conn, result)
-  end
-
-  def oauth_app_update(conn, params) do
-    params =
-      if params["name"] do
-        Map.put(params, "client_name", params["name"])
-      else
-        params
-      end
-
-    with {:ok, app} <- App.update(params) do
-      json(conn, AppView.render("show.json", %{app: app, admin: true}))
-    else
-      {:error, changeset} ->
-        json(conn, App.errors(changeset))
-
-      nil ->
-        json_response(conn, :bad_request, "")
-    end
-  end
-
-  def oauth_app_list(conn, params) do
-    {page, page_size} = page_params(params)
-
-    search_params = %{
-      client_name: params["name"],
-      client_id: params["client_id"],
-      page: page,
-      page_size: page_size
-    }
-
-    search_params =
-      if Map.has_key?(params, "trusted") do
-        Map.put(search_params, :trusted, params["trusted"])
-      else
-        search_params
-      end
-
-    with {:ok, apps, count} <- App.search(search_params) do
-      json(
-        conn,
-        AppView.render("index.json",
-          apps: apps,
-          count: count,
-          page_size: page_size,
-          admin: true
-        )
-      )
-    end
-  end
-
-  def oauth_app_delete(conn, params) do
-    with {:ok, _app} <- App.destroy(params["id"]) do
-      json_response(conn, :no_content, "")
-    else
-      _ -> json_response(conn, :bad_request, "")
-    end
-  end
-
   def stats(conn, params) do
     counters = Stats.get_status_visibility_count(params["instance"])
 
     json(conn, %{"status_visibility" => counters})
   end
 
-  defp errors(conn, {:error, :not_found}) do
-    conn
-    |> put_status(:not_found)
-    |> json(dgettext("errors", "Not found"))
-  end
-
-  defp errors(conn, {:error, reason}) do
-    conn
-    |> put_status(:bad_request)
-    |> json(reason)
-  end
-
-  defp errors(conn, {:param_cast, _}) do
-    conn
-    |> put_status(:bad_request)
-    |> json(dgettext("errors", "Invalid parameters"))
-  end
-
-  defp errors(conn, _) do
-    conn
-    |> put_status(:internal_server_error)
-    |> json(dgettext("errors", "Something went wrong"))
-  end
-
   defp page_params(params) do
     {get_page(params["page"]), get_page_size(params["page_size"])}
   end

lib/pleroma/web/admin_api/controllers/config_controller.ex

@@ -0,0 +1,152 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.ConfigController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Config
+  alias Pleroma.ConfigDB
+  alias Pleroma.Plugs.OAuthScopesPlug
+
+  @descriptions Pleroma.Docs.JSON.compile()
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+  plug(OAuthScopesPlug, %{scopes: ["write"], admin: true} when action == :update)
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["read"], admin: true}
+    when action in [:show, :descriptions]
+  )
+
+  action_fallback(Pleroma.Web.AdminAPI.FallbackController)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.ConfigOperation
+
+  def descriptions(conn, _params) do
+    descriptions = Enum.filter(@descriptions, &whitelisted_config?/1)
+
+    json(conn, descriptions)
+  end
+
+  def show(conn, %{only_db: true}) do
+    with :ok <- configurable_from_database() do
+      configs = Pleroma.Repo.all(ConfigDB)
+      render(conn, "index.json", %{configs: configs})
+    end
+  end
+
+  def show(conn, _params) do
+    with :ok <- configurable_from_database() do
+      configs = ConfigDB.get_all_as_keyword()
+
+      merged =
+        Config.Holder.default_config()
+        |> ConfigDB.merge(configs)
+        |> Enum.map(fn {group, values} ->
+          Enum.map(values, fn {key, value} ->
+            db =
+              if configs[group][key] do
+                ConfigDB.get_db_keys(configs[group][key], key)
+              end
+
+            db_value = configs[group][key]
+
+            merged_value =
+              if not is_nil(db_value) and Keyword.keyword?(db_value) and
+                   ConfigDB.sub_key_full_update?(group, key, Keyword.keys(db_value)) do
+                ConfigDB.merge_group(group, key, value, db_value)
+              else
+                value
+              end
+
+            %{
+              group: ConfigDB.convert(group),
+              key: ConfigDB.convert(key),
+              value: ConfigDB.convert(merged_value)
+            }
+            |> Pleroma.Maps.put_if_present(:db, db)
+          end)
+        end)
+        |> List.flatten()
+
+      json(conn, %{configs: merged, need_reboot: Restarter.Pleroma.need_reboot?()})
+    end
+  end
+
+  def update(%{body_params: %{configs: configs}} = conn, _) do
+    with :ok <- configurable_from_database() do
+      results =
+        configs
+        |> Enum.filter(&whitelisted_config?/1)
+        |> Enum.map(fn
+          %{group: group, key: key, delete: true} = params ->
+            ConfigDB.delete(%{group: group, key: key, subkeys: params[:subkeys]})
+
+          %{group: group, key: key, value: value} ->
+            ConfigDB.update_or_create(%{group: group, key: key, value: value})
+        end)
+        |> Enum.reject(fn {result, _} -> result == :error end)
+
+      {deleted, updated} =
+        results
+        |> Enum.map(fn {:ok, config} ->
+          Map.put(config, :db, ConfigDB.get_db_keys(config))
+        end)
+        |> Enum.split_with(fn config ->
+          Ecto.get_meta(config, :state) == :deleted
+        end)
+
+      Config.TransferTask.load_and_update_env(deleted, false)
+
+      if not Restarter.Pleroma.need_reboot?() do
+        changed_reboot_settings? =
+          (updated ++ deleted)
+          |> Enum.any?(fn config ->
+            group = ConfigDB.from_string(config.group)
+            key = ConfigDB.from_string(config.key)
+            value = ConfigDB.from_binary(config.value)
+            Config.TransferTask.pleroma_need_restart?(group, key, value)
+          end)
+
+        if changed_reboot_settings?, do: Restarter.Pleroma.need_reboot()
+      end
+
+      render(conn, "index.json", %{
+        configs: updated,
+        need_reboot: Restarter.Pleroma.need_reboot?()
+      })
+    end
+  end
+
+  defp configurable_from_database do
+    if Config.get(:configurable_from_database) do
+      :ok
+    else
+      {:error, "To use this endpoint you need to enable configuration from database."}
+    end
+  end
+
+  defp whitelisted_config?(group, key) do
+    if whitelisted_configs = Config.get(:database_config_whitelist) do
+      Enum.any?(whitelisted_configs, fn
+        {whitelisted_group} ->
+          group == inspect(whitelisted_group)
+
+        {whitelisted_group, whitelisted_key} ->
+          group == inspect(whitelisted_group) && key == inspect(whitelisted_key)
+      end)
+    else
+      true
+    end
+  end
+
+  defp whitelisted_config?(%{group: group, key: key}) do
+    whitelisted_config?(group, key)
+  end
+
+  defp whitelisted_config?(%{group: group} = config) do
+    whitelisted_config?(group, config[:key])
+  end
+end

lib/pleroma/web/admin_api/controllers/fallback_controller.ex

@@ -0,0 +1,31 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.FallbackController do
+  use Pleroma.Web, :controller
+
+  def call(conn, {:error, :not_found}) do
+    conn
+    |> put_status(:not_found)
+    |> json(%{error: dgettext("errors", "Not found")})
+  end
+
+  def call(conn, {:error, reason}) do
+    conn
+    |> put_status(:bad_request)
+    |> json(%{error: reason})
+  end
+
+  def call(conn, {:param_cast, _}) do
+    conn
+    |> put_status(:bad_request)
+    |> json(dgettext("errors", "Invalid parameters"))
+  end
+
+  def call(conn, _) do
+    conn
+    |> put_status(:internal_server_error)
+    |> json(%{error: dgettext("errors", "Something went wrong")})
+  end
+end

lib/pleroma/web/admin_api/controllers/invite_controller.ex

@@ -0,0 +1,78 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.InviteController do
+  use Pleroma.Web, :controller
+
+  import Pleroma.Web.ControllerHelper, only: [json_response: 3]
+
+  alias Pleroma.Config
+  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.UserInviteToken
+
+  require Logger
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+  plug(OAuthScopesPlug, %{scopes: ["read:invites"], admin: true} when action == :index)
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["write:invites"], admin: true} when action in [:create, :revoke, :email]
+  )
+
+  action_fallback(Pleroma.Web.AdminAPI.FallbackController)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.InviteOperation
+
+  @doc "Get list of created invites"
+  def index(conn, _params) do
+    invites = UserInviteToken.list_invites()
+
+    render(conn, "index.json", invites: invites)
+  end
+
+  @doc "Create an account registration invite token"
+  def create(%{body_params: params} = conn, _) do
+    {:ok, invite} = UserInviteToken.create_invite(params)
+
+    render(conn, "show.json", invite: invite)
+  end
+
+  @doc "Revokes invite by token"
+  def revoke(%{body_params: %{token: token}} = conn, _) do
+    with {:ok, invite} <- UserInviteToken.find_by_token(token),
+         {:ok, updated_invite} = UserInviteToken.update_invite(invite, %{used: true}) do
+      render(conn, "show.json", invite: updated_invite)
+    else
+      nil -> {:error, :not_found}
+      error -> error
+    end
+  end
+
+  @doc "Sends registration invite via email"
+  def email(%{assigns: %{user: user}, body_params: %{email: email} = params} = conn, _) do
+    with {_, false} <- {:registrations_open, Config.get([:instance, :registrations_open])},
+         {_, true} <- {:invites_enabled, Config.get([:instance, :invites_enabled])},
+         {:ok, invite_token} <- UserInviteToken.create_invite(),
+         {:ok, _} <-
+           user
+           |> Pleroma.Emails.UserEmail.user_invitation_email(
+             invite_token,
+             email,
+             params[:name]
+           )
+           |> Pleroma.Emails.Mailer.deliver() do
+      json_response(conn, :no_content, "")
+    else
+      {:registrations_open, _} ->
+        {:error, "To send invites you need to set the `registrations_open` option to false."}
+
+      {:invites_enabled, _} ->
+        {:error, "To send invites you need to set the `invites_enabled` option to true."}
+
+      {:error, error} ->
+        {:error, error}
+    end
+  end
+end

lib/pleroma/web/admin_api/controllers/oauth_app_controller.ex

@@ -0,0 +1,77 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.OAuthAppController do
+  use Pleroma.Web, :controller
+
+  import Pleroma.Web.ControllerHelper, only: [json_response: 3]
+
+  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.Web.OAuth.App
+
+  require Logger
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+  plug(:put_view, Pleroma.Web.MastodonAPI.AppView)
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["write"], admin: true}
+    when action in [:create, :index, :update, :delete]
+  )
+
+  action_fallback(Pleroma.Web.AdminAPI.FallbackController)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.OAuthAppOperation
+
+  def index(conn, params) do
+    search_params =
+      params
+      |> Map.take([:client_id, :page, :page_size, :trusted])
+      |> Map.put(:client_name, params[:name])
+
+    with {:ok, apps, count} <- App.search(search_params) do
+      render(conn, "index.json",
+        apps: apps,
+        count: count,
+        page_size: params.page_size,
+        admin: true
+      )
+    end
+  end
+
+  def create(%{body_params: params} = conn, _) do
+    params = Pleroma.Maps.put_if_present(params, :client_name, params[:name])
+
+    case App.create(params) do
+      {:ok, app} ->
+        render(conn, "show.json", app: app, admin: true)
+
+      {:error, changeset} ->
+        json(conn, App.errors(changeset))
+    end
+  end
+
+  def update(%{body_params: params} = conn, %{id: id}) do
+    params = Pleroma.Maps.put_if_present(params, :client_name, params[:name])
+
+    with {:ok, app} <- App.update(id, params) do
+      render(conn, "show.json", app: app, admin: true)
+    else
+      {:error, changeset} ->
+        json(conn, App.errors(changeset))
+
+      nil ->
+        json_response(conn, :bad_request, "")
+    end
+  end
+
+  def delete(conn, params) do
+    with {:ok, _app} <- App.destroy(params.id) do
+      json_response(conn, :no_content, "")
+    else
+      _ -> json_response(conn, :bad_request, "")
+    end
+  end
+end

lib/pleroma/web/admin_api/controllers/relay_controller.ex

@@ -0,0 +1,67 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.RelayController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.ModerationLog
+  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.Web.ActivityPub.Relay
+
+  require Logger
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["write:follows"], admin: true}
+    when action in [:follow, :unfollow]
+  )
+
+  plug(OAuthScopesPlug, %{scopes: ["read"], admin: true} when action == :index)
+
+  action_fallback(Pleroma.Web.AdminAPI.FallbackController)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.RelayOperation
+
+  def index(conn, _params) do
+    with {:ok, list} <- Relay.list() do
+      json(conn, %{relays: list})
+    end
+  end
+
+  def follow(%{assigns: %{user: admin}, body_params: %{relay_url: target}} = conn, _) do
+    with {:ok, _message} <- Relay.follow(target) do
+      ModerationLog.insert_log(%{
+        action: "relay_follow",
+        actor: admin,
+        target: target
+      })
+
+      json(conn, target)
+    else
+      _ ->
+        conn
+        |> put_status(500)
+        |> json(target)
+    end
+  end
+
+  def unfollow(%{assigns: %{user: admin}, body_params: %{relay_url: target}} = conn, _) do
+    with {:ok, _message} <- Relay.unfollow(target) do
+      ModerationLog.insert_log(%{
+        action: "relay_unfollow",
+        actor: admin,
+        target: target
+      })
+
+      json(conn, target)
+    else
+      _ ->
+        conn
+        |> put_status(500)
+        |> json(target)
+    end
+  end
+end

lib/pleroma/web/admin_api/controllers/report_controller.ex

@@ -0,0 +1,107 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.ReportController do
+  use Pleroma.Web, :controller
+
+  import Pleroma.Web.ControllerHelper, only: [json_response: 3]
+
+  alias Pleroma.Activity
+  alias Pleroma.ModerationLog
+  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.ReportNote
+  alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.Web.AdminAPI
+  alias Pleroma.Web.AdminAPI.Report
+  alias Pleroma.Web.CommonAPI
+
+  require Logger
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+  plug(OAuthScopesPlug, %{scopes: ["read:reports"], admin: true} when action in [:index, :show])
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["write:reports"], admin: true}
+    when action in [:update, :notes_create, :notes_delete]
+  )
+
+  action_fallback(AdminAPI.FallbackController)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.ReportOperation
+
+  def index(conn, params) do
+    reports = Utils.get_reports(params, params.page, params.page_size)
+
+    render(conn, "index.json", reports: reports)
+  end
+
+  def show(conn, %{id: id}) do
+    with %Activity{} = report <- Activity.get_by_id(id) do
+      render(conn, "show.json", Report.extract_report_info(report))
+    else
+      _ -> {:error, :not_found}
+    end
+  end
+
+  def update(%{assigns: %{user: admin}, body_params: %{reports: reports}} = conn, _) do
+    result =
+      Enum.map(reports, fn report ->
+        case CommonAPI.update_report_state(report.id, report.state) do
+          {:ok, activity} ->
+            ModerationLog.insert_log(%{
+              action: "report_update",
+              actor: admin,
+              subject: activity
+            })
+
+            activity
+
+          {:error, message} ->
+            %{id: report.id, error: message}
+        end
+      end)
+
+    if Enum.any?(result, &Map.has_key?(&1, :error)) do
+      json_response(conn, :bad_request, result)
+    else
+      json_response(conn, :no_content, "")
+    end
+  end
+
+  def notes_create(%{assigns: %{user: user}, body_params: %{content: content}} = conn, %{
+        id: report_id
+      }) do
+    with {:ok, _} <- ReportNote.create(user.id, report_id, content) do
+      ModerationLog.insert_log(%{
+        action: "report_note",
+        actor: user,
+        subject: Activity.get_by_id(report_id),
+        text: content
+      })
+
+      json_response(conn, :no_content, "")
+    else
+      _ -> json_response(conn, :bad_request, "")
+    end
+  end
+
+  def notes_delete(%{assigns: %{user: user}} = conn, %{
+        id: note_id,
+        report_id: report_id
+      }) do
+    with {:ok, note} <- ReportNote.destroy(note_id) do
+      ModerationLog.insert_log(%{
+        action: "report_note_delete",
+        actor: user,
+        subject: Activity.get_by_id(report_id),
+        text: note.content
+      })
+
+      json_response(conn, :no_content, "")
+    else
+      _ -> json_response(conn, :bad_request, "")
+    end
+  end
+end

lib/pleroma/web/admin_api/controllers/status_controller.ex

@@ -0,0 +1,77 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.StatusController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Activity
+  alias Pleroma.ModerationLog
+  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.MastodonAPI
+
+  require Logger
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+  plug(OAuthScopesPlug, %{scopes: ["read:statuses"], admin: true} when action in [:index, :show])
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["write:statuses"], admin: true} when action in [:update, :delete]
+  )
+
+  action_fallback(Pleroma.Web.AdminAPI.FallbackController)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.StatusOperation
+
+  def index(%{assigns: %{user: _admin}} = conn, params) do
+    activities =
+      ActivityPub.fetch_statuses(nil, %{
+        godmode: params.godmode,
+        local_only: params.local_only,
+        limit: params.page_size,
+        offset: (params.page - 1) * params.page_size,
+        exclude_reblogs: not params.with_reblogs
+      })
+
+    render(conn, "index.json", activities: activities, as: :activity)
+  end
+
+  def show(conn, %{id: id}) do
+    with %Activity{} = activity <- Activity.get_by_id(id) do
+      render(conn, "show.json", %{activity: activity})
+    else
+      nil -> {:error, :not_found}
+    end
+  end
+
+  def update(%{assigns: %{user: admin}, body_params: params} = conn, %{id: id}) do
+    with {:ok, activity} <- CommonAPI.update_activity_scope(id, params) do
+      ModerationLog.insert_log(%{
+        action: "status_update",
+        actor: admin,
+        subject: activity,
+        sensitive: params[:sensitive],
+        visibility: params[:visibility]
+      })
+
+      conn
+      |> put_view(MastodonAPI.StatusView)
+      |> render("show.json", %{activity: activity})
+    end
+  end
+
+  def delete(%{assigns: %{user: user}} = conn, %{id: id}) do
+    with {:ok, %Activity{}} <- CommonAPI.delete(id, user) do
+      ModerationLog.insert_log(%{
+        action: "status_delete",
+        actor: user,
+        subject_id: id
+      })
+
+      json(conn, %{})
+    end
+  end
+end

lib/pleroma/web/admin_api/search.ex

@@ -21,7 +21,7 @@ defmodule Pleroma.Web.AdminAPI.Search do
     query =
       params
       |> Map.drop([:page, :page_size])
-      |> Map.put(:exclude_service_users, true)
+      |> Map.put(:invisible, false)
       |> User.Query.build()
       |> order_by([u], u.nickname)
 
@@ -31,7 +31,6 @@ defmodule Pleroma.Web.AdminAPI.Search do
     count = Repo.aggregate(query, :count, :id)
 
     results = Repo.all(paginated_query)
-
     {:ok, results, count}
   end
 end

lib/pleroma/web/admin_api/views/account_view.ex

@@ -80,24 +80,6 @@ defmodule Pleroma.Web.AdminAPI.AccountView do
     }
   end
 
-  def render("invite.json", %{invite: invite}) do
-    %{
-      "id" => invite.id,
-      "token" => invite.token,
-      "used" => invite.used,
-      "expires_at" => invite.expires_at,
-      "uses" => invite.uses,
-      "max_use" => invite.max_use,
-      "invite_type" => invite.invite_type
-    }
-  end
-
-  def render("invites.json", %{invites: invites}) do
-    %{
-      invites: render_many(invites, AccountView, "invite.json", as: :invite)
-    }
-  end
-
   def render("created.json", %{user: user}) do
     %{
       type: "success",

lib/pleroma/web/admin_api/views/invite_view.ex

@@ -0,0 +1,25 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.InviteView do
+  use Pleroma.Web, :view
+
+  def render("index.json", %{invites: invites}) do
+    %{
+      invites: render_many(invites, __MODULE__, "show.json", as: :invite)
+    }
+  end
+
+  def render("show.json", %{invite: invite}) do
+    %{
+      "id" => invite.id,
+      "token" => invite.token,
+      "used" => invite.used,
+      "expires_at" => invite.expires_at,
+      "uses" => invite.uses,
+      "max_use" => invite.max_use,
+      "invite_type" => invite.invite_type
+    }
+  end
+end

lib/pleroma/web/admin_api/views/report_view.ex

@@ -18,7 +18,7 @@ defmodule Pleroma.Web.AdminAPI.ReportView do
     %{
       reports:
         reports[:items]
-        |> Enum.map(&Report.extract_report_info(&1))
+        |> Enum.map(&Report.extract_report_info/1)
         |> Enum.map(&render(__MODULE__, "show.json", &1))
         |> Enum.reverse(),
       total: reports[:total]

lib/pleroma/web/api_spec/operations/account_operation.ex

@@ -393,7 +393,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
           format: :password
         },
         agreement: %Schema{
-          type: :boolean,
+          allOf: [BooleanLike],
           description:
             "Whether the user agrees to the local rules, terms, and policies. These should be presented to the user in order to allow them to consent before setting this parameter to TRUE."
         },
@@ -463,7 +463,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
       type: :object,
       properties: %{
         bot: %Schema{
-          type: :boolean,
+          allOf: [BooleanLike],
           nullable: true,
           description: "Whether the account has a bot flag."
         },
@@ -486,7 +486,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
           format: :binary
         },
         locked: %Schema{
-          type: :boolean,
+          allOf: [BooleanLike],
           nullable: true,
           description: "Whether manual approval of follow requests is required."
         },
@@ -510,37 +510,37 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
 
         # Pleroma-specific fields
         no_rich_text: %Schema{
-          type: :boolean,
+          allOf: [BooleanLike],
           nullable: true,
           description: "html tags are stripped from all statuses requested from the API"
         },
         hide_followers: %Schema{
-          type: :boolean,
+          allOf: [BooleanLike],
           nullable: true,
           description: "user's followers will be hidden"
         },
         hide_follows: %Schema{
-          type: :boolean,
+          allOf: [BooleanLike],
           nullable: true,
           description: "user's follows will be hidden"
         },
         hide_followers_count: %Schema{
-          type: :boolean,
+          allOf: [BooleanLike],
           nullable: true,
           description: "user's follower count will be hidden"
         },
         hide_follows_count: %Schema{
-          type: :boolean,
+          allOf: [BooleanLike],
           nullable: true,
           description: "user's follow count will be hidden"
         },
         hide_favorites: %Schema{
-          type: :boolean,
+          allOf: [BooleanLike],
           nullable: true,
           description: "user's favorites timeline will be hidden"
         },
         show_role: %Schema{
-          type: :boolean,
+          allOf: [BooleanLike],
           nullable: true,
           description: "user's role (e.g admin, moderator) will be exposed to anyone in the
         API"
@@ -552,12 +552,12 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
           description: "Opaque user settings to be saved on the backend."
         },
         skip_thread_containment: %Schema{
-          type: :boolean,
+          allOf: [BooleanLike],
           nullable: true,
           description: "Skip filtering out broken threads"
         },
         allow_following_move: %Schema{
-          type: :boolean,
+          allOf: [BooleanLike],
           nullable: true,
           description: "Allows automatically follow moved following accounts"
         },
@@ -568,7 +568,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
           format: :binary
         },
         discoverable: %Schema{
-          type: :boolean,
+          allOf: [BooleanLike],
           nullable: true,
           description:
             "Discovery of this account in search results and other services is allowed."
@@ -678,7 +678,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
       type: :object,
       properties: %{
         notifications: %Schema{
-          type: :boolean,
+          allOf: [BooleanLike],
           nullable: true,
           description: "Mute notifications in addition to statuses? Defaults to true.",
           default: true

lib/pleroma/web/api_spec/operations/admin/config_operation.ex

@@ -0,0 +1,142 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.ConfigOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def show_operation do
+    %Operation{
+      tags: ["Admin", "Config"],
+      summary: "Get list of merged default settings with saved in database",
+      operationId: "AdminAPI.ConfigController.show",
+      parameters: [
+        Operation.parameter(
+          :only_db,
+          :query,
+          %Schema{type: :boolean, default: false},
+          "Get only saved in database settings"
+        )
+      ],
+      security: [%{"oAuth" => ["read"]}],
+      responses: %{
+        200 => Operation.response("Config", "application/json", config_response()),
+        400 => Operation.response("Bad Request", "application/json", ApiError)
+      }
+    }
+  end
+
+  def update_operation do
+    %Operation{
+      tags: ["Admin", "Config"],
+      summary: "Update config settings",
+      operationId: "AdminAPI.ConfigController.update",
+      security: [%{"oAuth" => ["write"]}],
+      requestBody:
+        request_body("Parameters", %Schema{
+          type: :object,
+          properties: %{
+            configs: %Schema{
+              type: :array,
+              items: %Schema{
+                type: :object,
+                properties: %{
+                  group: %Schema{type: :string},
+                  key: %Schema{type: :string},
+                  value: any(),
+                  delete: %Schema{type: :boolean},
+                  subkeys: %Schema{type: :array, items: %Schema{type: :string}}
+                }
+              }
+            }
+          }
+        }),
+      responses: %{
+        200 => Operation.response("Config", "application/json", config_response()),
+        400 => Operation.response("Bad Request", "application/json", ApiError)
+      }
+    }
+  end
+
+  def descriptions_operation do
+    %Operation{
+      tags: ["Admin", "Config"],
+      summary: "Get JSON with config descriptions.",
+      operationId: "AdminAPI.ConfigController.descriptions",
+      security: [%{"oAuth" => ["read"]}],
+      responses: %{
+        200 =>
+          Operation.response("Config Descriptions", "application/json", %Schema{
+            type: :array,
+            items: %Schema{
+              type: :object,
+              properties: %{
+                group: %Schema{type: :string},
+                key: %Schema{type: :string},
+                type: %Schema{oneOf: [%Schema{type: :string}, %Schema{type: :array}]},
+                description: %Schema{type: :string},
+                children: %Schema{
+                  type: :array,
+                  items: %Schema{
+                    type: :object,
+                    properties: %{
+                      key: %Schema{type: :string},
+                      type: %Schema{oneOf: [%Schema{type: :string}, %Schema{type: :array}]},
+                      description: %Schema{type: :string},
+                      suggestions: %Schema{type: :array}
+                    }
+                  }
+                }
+              }
+            }
+          }),
+        400 => Operation.response("Bad Request", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp any do
+    %Schema{
+      oneOf: [
+        %Schema{type: :array},
+        %Schema{type: :object},
+        %Schema{type: :string},
+        %Schema{type: :integer},
+        %Schema{type: :boolean}
+      ]
+    }
+  end
+
+  defp config_response do
+    %Schema{
+      type: :object,
+      properties: %{
+        configs: %Schema{
+          type: :array,
+          items: %Schema{
+            type: :object,
+            properties: %{
+              group: %Schema{type: :string},
+              key: %Schema{type: :string},
+              value: any()
+            }
+          }
+        },
+        need_reboot: %Schema{
+          type: :boolean,
+          description:
+            "If `need_reboot` is `true`, instance must be restarted, so reboot time settings can take effect"
+        }
+      }
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/admin/invite_operation.ex

@@ -0,0 +1,148 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.InviteOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def index_operation do
+    %Operation{
+      tags: ["Admin", "Invites"],
+      summary: "Get a list of generated invites",
+      operationId: "AdminAPI.InviteController.index",
+      security: [%{"oAuth" => ["read:invites"]}],
+      responses: %{
+        200 =>
+          Operation.response("Invites", "application/json", %Schema{
+            type: :object,
+            properties: %{
+              invites: %Schema{type: :array, items: invite()}
+            },
+            example: %{
+              "invites" => [
+                %{
+                  "id" => 123,
+                  "token" => "kSQtDj_GNy2NZsL9AQDFIsHN5qdbguB6qRg3WHw6K1U=",
+                  "used" => true,
+                  "expires_at" => nil,
+                  "uses" => 0,
+                  "max_use" => nil,
+                  "invite_type" => "one_time"
+                }
+              ]
+            }
+          })
+      }
+    }
+  end
+
+  def create_operation do
+    %Operation{
+      tags: ["Admin", "Invites"],
+      summary: "Create an account registration invite token",
+      operationId: "AdminAPI.InviteController.create",
+      security: [%{"oAuth" => ["write:invites"]}],
+      requestBody:
+        request_body("Parameters", %Schema{
+          type: :object,
+          properties: %{
+            max_use: %Schema{type: :integer},
+            expires_at: %Schema{type: :string, format: :date, example: "2020-04-20"}
+          }
+        }),
+      responses: %{
+        200 => Operation.response("Invite", "application/json", invite())
+      }
+    }
+  end
+
+  def revoke_operation do
+    %Operation{
+      tags: ["Admin", "Invites"],
+      summary: "Revoke invite by token",
+      operationId: "AdminAPI.InviteController.revoke",
+      security: [%{"oAuth" => ["write:invites"]}],
+      requestBody:
+        request_body(
+          "Parameters",
+          %Schema{
+            type: :object,
+            required: [:token],
+            properties: %{
+              token: %Schema{type: :string}
+            }
+          },
+          required: true
+        ),
+      responses: %{
+        200 => Operation.response("Invite", "application/json", invite()),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  def email_operation do
+    %Operation{
+      tags: ["Admin", "Invites"],
+      summary: "Sends registration invite via email",
+      operationId: "AdminAPI.InviteController.email",
+      security: [%{"oAuth" => ["write:invites"]}],
+      requestBody:
+        request_body(
+          "Parameters",
+          %Schema{
+            type: :object,
+            required: [:email],
+            properties: %{
+              email: %Schema{type: :string, format: :email},
+              name: %Schema{type: :string}
+            }
+          },
+          required: true
+        ),
+      responses: %{
+        204 => no_content_response(),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp invite do
+    %Schema{
+      title: "Invite",
+      type: :object,
+      properties: %{
+        id: %Schema{type: :integer},
+        token: %Schema{type: :string},
+        used: %Schema{type: :boolean},
+        expires_at: %Schema{type: :string, format: :date, nullable: true},
+        uses: %Schema{type: :integer},
+        max_use: %Schema{type: :integer, nullable: true},
+        invite_type: %Schema{
+          type: :string,
+          enum: ["one_time", "reusable", "date_limited", "reusable_date_limited"]
+        }
+      },
+      example: %{
+        "id" => 123,
+        "token" => "kSQtDj_GNy2NZsL9AQDFIsHN5qdbguB6qRg3WHw6K1U=",
+        "used" => true,
+        "expires_at" => nil,
+        "uses" => 0,
+        "max_use" => nil,
+        "invite_type" => "one_time"
+      }
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/admin/oauth_app_operation.ex

@@ -0,0 +1,215 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.OAuthAppOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def index_operation do
+    %Operation{
+      summary: "List OAuth apps",
+      tags: ["Admin", "oAuth Apps"],
+      operationId: "AdminAPI.OAuthAppController.index",
+      security: [%{"oAuth" => ["write"]}],
+      parameters: [
+        Operation.parameter(:name, :query, %Schema{type: :string}, "App name"),
+        Operation.parameter(:client_id, :query, %Schema{type: :string}, "Client ID"),
+        Operation.parameter(:page, :query, %Schema{type: :integer, default: 1}, "Page"),
+        Operation.parameter(
+          :trusted,
+          :query,
+          %Schema{type: :boolean, default: false},
+          "Trusted apps"
+        ),
+        Operation.parameter(
+          :page_size,
+          :query,
+          %Schema{type: :integer, default: 50},
+          "Number of apps to return"
+        )
+      ],
+      responses: %{
+        200 =>
+          Operation.response("List of apps", "application/json", %Schema{
+            type: :object,
+            properties: %{
+              apps: %Schema{type: :array, items: oauth_app()},
+              count: %Schema{type: :integer},
+              page_size: %Schema{type: :integer}
+            },
+            example: %{
+              "apps" => [
+                %{
+                  "id" => 1,
+                  "name" => "App name",
+                  "client_id" => "yHoDSiWYp5mPV6AfsaVOWjdOyt5PhWRiafi6MRd1lSk",
+                  "client_secret" => "nLmis486Vqrv2o65eM9mLQx_m_4gH-Q6PcDpGIMl6FY",
+                  "redirect_uri" => "https://example.com/oauth-callback",
+                  "website" => "https://example.com",
+                  "trusted" => true
+                }
+              ],
+              "count" => 1,
+              "page_size" => 50
+            }
+          })
+      }
+    }
+  end
+
+  def create_operation do
+    %Operation{
+      tags: ["Admin", "oAuth Apps"],
+      summary: "Create OAuth App",
+      operationId: "AdminAPI.OAuthAppController.create",
+      requestBody: request_body("Parameters", create_request()),
+      security: [%{"oAuth" => ["write"]}],
+      responses: %{
+        200 => Operation.response("App", "application/json", oauth_app()),
+        400 => Operation.response("Bad Request", "application/json", ApiError)
+      }
+    }
+  end
+
+  def update_operation do
+    %Operation{
+      tags: ["Admin", "oAuth Apps"],
+      summary: "Update OAuth App",
+      operationId: "AdminAPI.OAuthAppController.update",
+      parameters: [id_param()],
+      security: [%{"oAuth" => ["write"]}],
+      requestBody: request_body("Parameters", update_request()),
+      responses: %{
+        200 => Operation.response("App", "application/json", oauth_app()),
+        400 =>
+          Operation.response("Bad Request", "application/json", %Schema{
+            oneOf: [ApiError, %Schema{type: :string}]
+          })
+      }
+    }
+  end
+
+  def delete_operation do
+    %Operation{
+      tags: ["Admin", "oAuth Apps"],
+      summary: "Delete OAuth App",
+      operationId: "AdminAPI.OAuthAppController.delete",
+      parameters: [id_param()],
+      security: [%{"oAuth" => ["write"]}],
+      responses: %{
+        204 => no_content_response(),
+        400 => no_content_response()
+      }
+    }
+  end
+
+  defp create_request do
+    %Schema{
+      title: "oAuthAppCreateRequest",
+      type: :object,
+      required: [:name, :redirect_uris],
+      properties: %{
+        name: %Schema{type: :string, description: "Application Name"},
+        scopes: %Schema{type: :array, items: %Schema{type: :string}, description: "oAuth scopes"},
+        redirect_uris: %Schema{
+          type: :string,
+          description:
+            "Where the user should be redirected after authorization. To display the authorization code to the user instead of redirecting to a web page, use `urn:ietf:wg:oauth:2.0:oob` in this parameter."
+        },
+        website: %Schema{
+          type: :string,
+          nullable: true,
+          description: "A URL to the homepage of the app"
+        },
+        trusted: %Schema{
+          type: :boolean,
+          nullable: true,
+          default: false,
+          description: "Is the app trusted?"
+        }
+      },
+      example: %{
+        "name" => "My App",
+        "redirect_uris" => "https://myapp.com/auth/callback",
+        "website" => "https://myapp.com/",
+        "scopes" => ["read", "write"],
+        "trusted" => true
+      }
+    }
+  end
+
+  defp update_request do
+    %Schema{
+      title: "oAuthAppUpdateRequest",
+      type: :object,
+      properties: %{
+        name: %Schema{type: :string, description: "Application Name"},
+        scopes: %Schema{type: :array, items: %Schema{type: :string}, description: "oAuth scopes"},
+        redirect_uris: %Schema{
+          type: :string,
+          description:
+            "Where the user should be redirected after authorization. To display the authorization code to the user instead of redirecting to a web page, use `urn:ietf:wg:oauth:2.0:oob` in this parameter."
+        },
+        website: %Schema{
+          type: :string,
+          nullable: true,
+          description: "A URL to the homepage of the app"
+        },
+        trusted: %Schema{
+          type: :boolean,
+          nullable: true,
+          default: false,
+          description: "Is the app trusted?"
+        }
+      },
+      example: %{
+        "name" => "My App",
+        "redirect_uris" => "https://myapp.com/auth/callback",
+        "website" => "https://myapp.com/",
+        "scopes" => ["read", "write"],
+        "trusted" => true
+      }
+    }
+  end
+
+  defp oauth_app do
+    %Schema{
+      title: "oAuthApp",
+      type: :object,
+      properties: %{
+        id: %Schema{type: :integer},
+        name: %Schema{type: :string},
+        client_id: %Schema{type: :string},
+        client_secret: %Schema{type: :string},
+        redirect_uri: %Schema{type: :string},
+        website: %Schema{type: :string, nullable: true},
+        trusted: %Schema{type: :boolean}
+      },
+      example: %{
+        "id" => 123,
+        "name" => "My App",
+        "client_id" => "TWhM-tNSuncnqN7DBJmoyeLnk6K3iJJ71KKXxgL1hPM",
+        "client_secret" => "ZEaFUFmF0umgBX1qKJDjaU99Q31lDkOU8NutzTOoliw",
+        "redirect_uri" => "https://myapp.com/oauth-callback",
+        "website" => "https://myapp.com/",
+        "trusted" => false
+      }
+    }
+  end
+
+  def id_param do
+    Operation.parameter(:id, :path, :integer, "App ID",
+      example: 1337,
+      required: true
+    )
+  end
+end

lib/pleroma/web/api_spec/operations/admin/relay_operation.ex

@@ -0,0 +1,83 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.RelayOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def index_operation do
+    %Operation{
+      tags: ["Admin", "Relays"],
+      summary: "List Relays",
+      operationId: "AdminAPI.RelayController.index",
+      security: [%{"oAuth" => ["read"]}],
+      responses: %{
+        200 =>
+          Operation.response("Response", "application/json", %Schema{
+            type: :object,
+            properties: %{
+              relays: %Schema{
+                type: :array,
+                items: %Schema{type: :string},
+                example: ["lain.com", "mstdn.io"]
+              }
+            }
+          })
+      }
+    }
+  end
+
+  def follow_operation do
+    %Operation{
+      tags: ["Admin", "Relays"],
+      summary: "Follow a Relay",
+      operationId: "AdminAPI.RelayController.follow",
+      security: [%{"oAuth" => ["write:follows"]}],
+      requestBody:
+        request_body("Parameters", %Schema{
+          type: :object,
+          properties: %{
+            relay_url: %Schema{type: :string, format: :uri}
+          }
+        }),
+      responses: %{
+        200 =>
+          Operation.response("Status", "application/json", %Schema{
+            type: :string,
+            example: "http://mastodon.example.org/users/admin"
+          })
+      }
+    }
+  end
+
+  def unfollow_operation do
+    %Operation{
+      tags: ["Admin", "Relays"],
+      summary: "Unfollow a Relay",
+      operationId: "AdminAPI.RelayController.unfollow",
+      security: [%{"oAuth" => ["write:follows"]}],
+      requestBody:
+        request_body("Parameters", %Schema{
+          type: :object,
+          properties: %{
+            relay_url: %Schema{type: :string, format: :uri}
+          }
+        }),
+      responses: %{
+        200 =>
+          Operation.response("Status", "application/json", %Schema{
+            type: :string,
+            example: "http://mastodon.example.org/users/admin"
+          })
+      }
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/admin/report_operation.ex

@@ -0,0 +1,237 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.Account
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+  alias Pleroma.Web.ApiSpec.Schemas.FlakeID
+  alias Pleroma.Web.ApiSpec.Schemas.Status
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def index_operation do
+    %Operation{
+      tags: ["Admin", "Reports"],
+      summary: "Get a list of reports",
+      operationId: "AdminAPI.ReportController.index",
+      security: [%{"oAuth" => ["read:reports"]}],
+      parameters: [
+        Operation.parameter(
+          :state,
+          :query,
+          report_state(),
+          "Filter by report state"
+        ),
+        Operation.parameter(
+          :limit,
+          :query,
+          %Schema{type: :integer},
+          "The number of records to retrieve"
+        ),
+        Operation.parameter(
+          :page,
+          :query,
+          %Schema{type: :integer, default: 1},
+          "Page number"
+        ),
+        Operation.parameter(
+          :page_size,
+          :query,
+          %Schema{type: :integer, default: 50},
+          "Number number of log entries per page"
+        )
+      ],
+      responses: %{
+        200 =>
+          Operation.response("Response", "application/json", %Schema{
+            type: :object,
+            properties: %{
+              total: %Schema{type: :integer},
+              reports: %Schema{
+                type: :array,
+                items: report()
+              }
+            }
+          }),
+        403 => Operation.response("Forbidden", "application/json", ApiError)
+      }
+    }
+  end
+
+  def show_operation do
+    %Operation{
+      tags: ["Admin", "Reports"],
+      summary: "Get an individual report",
+      operationId: "AdminAPI.ReportController.show",
+      parameters: [id_param()],
+      security: [%{"oAuth" => ["read:reports"]}],
+      responses: %{
+        200 => Operation.response("Report", "application/json", report()),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  def update_operation do
+    %Operation{
+      tags: ["Admin", "Reports"],
+      summary: "Change the state of one or multiple reports",
+      operationId: "AdminAPI.ReportController.update",
+      security: [%{"oAuth" => ["write:reports"]}],
+      requestBody: request_body("Parameters", update_request(), required: true),
+      responses: %{
+        204 => no_content_response(),
+        400 => Operation.response("Bad Request", "application/json", update_400_response()),
+        403 => Operation.response("Forbidden", "application/json", ApiError)
+      }
+    }
+  end
+
+  def notes_create_operation do
+    %Operation{
+      tags: ["Admin", "Reports"],
+      summary: "Create report note",
+      operationId: "AdminAPI.ReportController.notes_create",
+      parameters: [id_param()],
+      requestBody:
+        request_body("Parameters", %Schema{
+          type: :object,
+          properties: %{
+            content: %Schema{type: :string, description: "The message"}
+          }
+        }),
+      security: [%{"oAuth" => ["write:reports"]}],
+      responses: %{
+        204 => no_content_response(),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  def notes_delete_operation do
+    %Operation{
+      tags: ["Admin", "Reports"],
+      summary: "Delete report note",
+      operationId: "AdminAPI.ReportController.notes_delete",
+      parameters: [
+        Operation.parameter(:report_id, :path, :string, "Report ID"),
+        Operation.parameter(:id, :path, :string, "Note ID")
+      ],
+      security: [%{"oAuth" => ["write:reports"]}],
+      responses: %{
+        204 => no_content_response(),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp report_state do
+    %Schema{type: :string, enum: ["open", "closed", "resolved"]}
+  end
+
+  defp id_param do
+    Operation.parameter(:id, :path, FlakeID, "Report ID",
+      example: "9umDrYheeY451cQnEe",
+      required: true
+    )
+  end
+
+  defp report do
+    %Schema{
+      type: :object,
+      properties: %{
+        id: FlakeID,
+        state: report_state(),
+        account: account_admin(),
+        actor: account_admin(),
+        content: %Schema{type: :string},
+        created_at: %Schema{type: :string, format: :"date-time"},
+        statuses: %Schema{type: :array, items: Status},
+        notes: %Schema{
+          type: :array,
+          items: %Schema{
+            type: :object,
+            properties: %{
+              id: %Schema{type: :integer},
+              user_id: FlakeID,
+              content: %Schema{type: :string},
+              inserted_at: %Schema{type: :string, format: :"date-time"}
+            }
+          }
+        }
+      }
+    }
+  end
+
+  defp account_admin do
+    %Schema{
+      title: "Account",
+      description: "Account view for admins",
+      type: :object,
+      properties:
+        Map.merge(Account.schema().properties, %{
+          nickname: %Schema{type: :string},
+          deactivated: %Schema{type: :boolean},
+          local: %Schema{type: :boolean},
+          roles: %Schema{
+            type: :object,
+            properties: %{
+              admin: %Schema{type: :boolean},
+              moderator: %Schema{type: :boolean}
+            }
+          },
+          confirmation_pending: %Schema{type: :boolean}
+        })
+    }
+  end
+
+  defp update_request do
+    %Schema{
+      type: :object,
+      required: [:reports],
+      properties: %{
+        reports: %Schema{
+          type: :array,
+          items: %Schema{
+            type: :object,
+            properties: %{
+              id: %Schema{allOf: [FlakeID], description: "Required, report ID"},
+              state: %Schema{
+                type: :string,
+                description:
+                  "Required, the new state. Valid values are `open`, `closed` and `resolved`"
+              }
+            }
+          },
+          example: %{
+            "reports" => [
+              %{"id" => "123", "state" => "closed"},
+              %{"id" => "1337", "state" => "resolved"}
+            ]
+          }
+        }
+      }
+    }
+  end
+
+  defp update_400_response do
+    %Schema{
+      type: :array,
+      items: %Schema{
+        type: :object,
+        properties: %{
+          id: %Schema{allOf: [FlakeID], description: "Report ID"},
+          error: %Schema{type: :string, description: "Error message"}
+        }
+      }
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/admin/status_operation.ex

@@ -0,0 +1,165 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.StatusOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.Account
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+  alias Pleroma.Web.ApiSpec.Schemas.FlakeID
+  alias Pleroma.Web.ApiSpec.Schemas.Status
+  alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope
+
+  import Pleroma.Web.ApiSpec.Helpers
+  import Pleroma.Web.ApiSpec.StatusOperation, only: [id_param: 0]
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def index_operation do
+    %Operation{
+      tags: ["Admin", "Statuses"],
+      operationId: "AdminAPI.StatusController.index",
+      security: [%{"oAuth" => ["read:statuses"]}],
+      parameters: [
+        Operation.parameter(
+          :godmode,
+          :query,
+          %Schema{type: :boolean, default: false},
+          "Allows to see private statuses"
+        ),
+        Operation.parameter(
+          :local_only,
+          :query,
+          %Schema{type: :boolean, default: false},
+          "Excludes remote statuses"
+        ),
+        Operation.parameter(
+          :with_reblogs,
+          :query,
+          %Schema{type: :boolean, default: false},
+          "Allows to see reblogs"
+        ),
+        Operation.parameter(
+          :page,
+          :query,
+          %Schema{type: :integer, default: 1},
+          "Page"
+        ),
+        Operation.parameter(
+          :page_size,
+          :query,
+          %Schema{type: :integer, default: 50},
+          "Number of statuses to return"
+        )
+      ],
+      responses: %{
+        200 =>
+          Operation.response("Array of statuses", "application/json", %Schema{
+            type: :array,
+            items: status()
+          })
+      }
+    }
+  end
+
+  def show_operation do
+    %Operation{
+      tags: ["Admin", "Statuses"],
+      summary: "Show Status",
+      operationId: "AdminAPI.StatusController.show",
+      parameters: [id_param()],
+      security: [%{"oAuth" => ["read:statuses"]}],
+      responses: %{
+        200 => Operation.response("Status", "application/json", status()),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  def update_operation do
+    %Operation{
+      tags: ["Admin", "Statuses"],
+      summary: "Change the scope of an individual reported status",
+      operationId: "AdminAPI.StatusController.update",
+      parameters: [id_param()],
+      security: [%{"oAuth" => ["write:statuses"]}],
+      requestBody: request_body("Parameters", update_request(), required: true),
+      responses: %{
+        200 => Operation.response("Status", "application/json", Status),
+        400 => Operation.response("Error", "application/json", ApiError)
+      }
+    }
+  end
+
+  def delete_operation do
+    %Operation{
+      tags: ["Admin", "Statuses"],
+      summary: "Delete an individual reported status",
+      operationId: "AdminAPI.StatusController.delete",
+      parameters: [id_param()],
+      security: [%{"oAuth" => ["write:statuses"]}],
+      responses: %{
+        200 => empty_object_response(),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp status do
+    %Schema{
+      anyOf: [
+        Status,
+        %Schema{
+          type: :object,
+          properties: %{
+            account: %Schema{allOf: [Account, admin_account()]}
+          }
+        }
+      ]
+    }
+  end
+
+  def admin_account do
+    %Schema{
+      type: :object,
+      properties: %{
+        id: FlakeID,
+        avatar: %Schema{type: :string},
+        nickname: %Schema{type: :string},
+        display_name: %Schema{type: :string},
+        deactivated: %Schema{type: :boolean},
+        local: %Schema{type: :boolean},
+        roles: %Schema{
+          type: :object,
+          properties: %{
+            admin: %Schema{type: :boolean},
+            moderator: %Schema{type: :boolean}
+          }
+        },
+        tags: %Schema{type: :string},
+        confirmation_pending: %Schema{type: :string}
+      }
+    }
+  end
+
+  defp update_request do
+    %Schema{
+      type: :object,
+      properties: %{
+        sensitive: %Schema{
+          type: :boolean,
+          description: "Mark status and attached media as sensitive?"
+        },
+        visibility: VisibilityScope
+      },
+      example: %{
+        "visibility" => "private",
+        "sensitive" => "false"
+      }
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/emoji_reaction_operation.ex

@@ -0,0 +1,104 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.EmojiReactionOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.Account
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+  alias Pleroma.Web.ApiSpec.Schemas.FlakeID
+  alias Pleroma.Web.ApiSpec.Schemas.Status
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def index_operation do
+    %Operation{
+      tags: ["Emoji Reactions"],
+      summary:
+        "Get an object of emoji to account mappings with accounts that reacted to the post",
+      parameters: [
+        Operation.parameter(:id, :path, FlakeID, "Status ID", required: true),
+        Operation.parameter(:emoji, :path, :string, "Filter by a single unicode emoji",
+          required: false
+        )
+      ],
+      security: [%{"oAuth" => ["read:statuses"]}],
+      operationId: "EmojiReactionController.index",
+      responses: %{
+        200 => array_of_reactions_response()
+      }
+    }
+  end
+
+  def create_operation do
+    %Operation{
+      tags: ["Emoji Reactions"],
+      summary: "React to a post with a unicode emoji",
+      parameters: [
+        Operation.parameter(:id, :path, FlakeID, "Status ID", required: true),
+        Operation.parameter(:emoji, :path, :string, "A single character unicode emoji",
+          required: true
+        )
+      ],
+      security: [%{"oAuth" => ["write:statuses"]}],
+      operationId: "EmojiReactionController.create",
+      responses: %{
+        200 => Operation.response("Status", "application/json", Status),
+        400 => Operation.response("Bad Request", "application/json", ApiError)
+      }
+    }
+  end
+
+  def delete_operation do
+    %Operation{
+      tags: ["Emoji Reactions"],
+      summary: "Remove a reaction to a post with a unicode emoji",
+      parameters: [
+        Operation.parameter(:id, :path, FlakeID, "Status ID", required: true),
+        Operation.parameter(:emoji, :path, :string, "A single character unicode emoji",
+          required: true
+        )
+      ],
+      security: [%{"oAuth" => ["write:statuses"]}],
+      operationId: "EmojiReactionController.delete",
+      responses: %{
+        200 => Operation.response("Status", "application/json", Status)
+      }
+    }
+  end
+
+  defp array_of_reactions_response do
+    Operation.response("Array of Emoji Reactions", "application/json", %Schema{
+      type: :array,
+      items: emoji_reaction(),
+      example: [emoji_reaction().example]
+    })
+  end
+
+  defp emoji_reaction do
+    %Schema{
+      title: "EmojiReaction",
+      type: :object,
+      properties: %{
+        name: %Schema{type: :string, description: "Emoji"},
+        count: %Schema{type: :integer, description: "Count of reactions with this emoji"},
+        me: %Schema{type: :boolean, description: "Did I react with this emoji?"},
+        accounts: %Schema{
+          type: :array,
+          items: Account,
+          description: "Array of accounts reacted with this emoji"
+        }
+      },
+      example: %{
+        "name" => "😱",
+        "count" => 1,
+        "me" => false,
+        "accounts" => [Account.schema().example]
+      }
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/filter_operation.ex

@@ -6,6 +6,7 @@ defmodule Pleroma.Web.ApiSpec.FilterOperation do
   alias OpenApiSpex.Operation
   alias OpenApiSpex.Schema
   alias Pleroma.Web.ApiSpec.Helpers
+  alias Pleroma.Web.ApiSpec.Schemas.BooleanLike
 
   def open_api_operation(action) do
     operation = String.to_existing_atom("#{action}_operation")
@@ -171,7 +172,7 @@ defmodule Pleroma.Web.ApiSpec.FilterOperation do
           type: :object,
           properties: %{
             irreversible: %Schema{
-              type: :bolean,
+              allOf: [BooleanLike],
               description:
                 "Should the server irreversibly drop matching entities from home and notifications?",
               default: false
@@ -199,13 +200,13 @@ defmodule Pleroma.Web.ApiSpec.FilterOperation do
             "Array of enumerable strings `home`, `notifications`, `public`, `thread`. At least one context must be specified."
         },
         irreversible: %Schema{
-          type: :boolean,
+          allOf: [BooleanLike],
           nullable: true,
           description:
             "Should the server irreversibly drop matching entities from home and notifications?"
         },
         whole_word: %Schema{
-          type: :boolean,
+          allOf: [BooleanLike],
           nullable: true,
           description: "Consider word boundaries?",
           default: true

lib/pleroma/web/api_spec/operations/instance_operation.ex

@@ -125,13 +125,19 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do
         },
         avatar_upload_limit: %Schema{type: :integer, description: "The title of the website"},
         background_upload_limit: %Schema{type: :integer, description: "The title of the website"},
-        banner_upload_limit: %Schema{type: :integer, description: "The title of the website"}
+        banner_upload_limit: %Schema{type: :integer, description: "The title of the website"},
+        background_image: %Schema{
+          type: :string,
+          format: :uri,
+          description: "The background image for the website"
+        }
       },
       example: %{
         "avatar_upload_limit" => 2_000_000,
         "background_upload_limit" => 4_000_000,
+        "background_image" => "/static/image.png",
         "banner_upload_limit" => 4_000_000,
-        "description" => "A Pleroma instance, an alternative fediverse server",
+        "description" => "Pleroma: An efficient and flexible fediverse server",
         "email" => "lain@lain.com",
         "languages" => ["en"],
         "max_toot_chars" => 5000,

lib/pleroma/web/api_spec/operations/notification_operation.ex

@@ -145,7 +145,7 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do
     }
   end
 
-  defp notification do
+  def notification do
     %Schema{
       title: "Notification",
       description: "Response schema for a notification",

lib/pleroma/web/api_spec/operations/pleroma_conversation_operation.ex

@@ -0,0 +1,106 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.PleromaConversationOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.Conversation
+  alias Pleroma.Web.ApiSpec.Schemas.FlakeID
+  alias Pleroma.Web.ApiSpec.StatusOperation
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def show_operation do
+    %Operation{
+      tags: ["Conversations"],
+      summary: "The conversation with the given ID",
+      parameters: [
+        Operation.parameter(:id, :path, :string, "Conversation ID",
+          example: "123",
+          required: true
+        )
+      ],
+      security: [%{"oAuth" => ["read:statuses"]}],
+      operationId: "PleromaAPI.ConversationController.show",
+      responses: %{
+        200 => Operation.response("Conversation", "application/json", Conversation)
+      }
+    }
+  end
+
+  def statuses_operation do
+    %Operation{
+      tags: ["Conversations"],
+      summary: "Timeline for a given conversation",
+      parameters: [
+        Operation.parameter(:id, :path, :string, "Conversation ID",
+          example: "123",
+          required: true
+        )
+        | pagination_params()
+      ],
+      security: [%{"oAuth" => ["read:statuses"]}],
+      operationId: "PleromaAPI.ConversationController.statuses",
+      responses: %{
+        200 =>
+          Operation.response(
+            "Array of Statuses",
+            "application/json",
+            StatusOperation.array_of_statuses()
+          )
+      }
+    }
+  end
+
+  def update_operation do
+    %Operation{
+      tags: ["Conversations"],
+      summary: "Update a conversation. Used to change the set of recipients.",
+      parameters: [
+        Operation.parameter(:id, :path, :string, "Conversation ID",
+          example: "123",
+          required: true
+        ),
+        Operation.parameter(
+          :recipients,
+          :query,
+          %Schema{type: :array, items: FlakeID},
+          "A list of ids of users that should receive posts to this conversation. This will replace the current list of recipients, so submit the full list. The owner of owner of the conversation will always be part of the set of recipients, though.",
+          required: true
+        )
+      ],
+      security: [%{"oAuth" => ["write:conversations"]}],
+      operationId: "PleromaAPI.ConversationController.update",
+      responses: %{
+        200 => Operation.response("Conversation", "application/json", Conversation)
+      }
+    }
+  end
+
+  def mark_as_read_operation do
+    %Operation{
+      tags: ["Conversations"],
+      summary: "Marks all user's conversations as read",
+      security: [%{"oAuth" => ["write:conversations"]}],
+      operationId: "PleromaAPI.ConversationController.mark_as_read",
+      responses: %{
+        200 =>
+          Operation.response(
+            "Array of Conversations that were marked as read",
+            "application/json",
+            %Schema{
+              type: :array,
+              items: Conversation,
+              example: [Conversation.schema().example]
+            }
+          )
+      }
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex

@@ -0,0 +1,390 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.PleromaEmojiPackOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def remote_operation do
+    %Operation{
+      tags: ["Emoji Packs"],
+      summary: "Make request to another instance for emoji packs list",
+      security: [%{"oAuth" => ["write"]}],
+      parameters: [url_param()],
+      operationId: "PleromaAPI.EmojiPackController.remote",
+      responses: %{
+        200 => emoji_packs_response(),
+        500 => Operation.response("Error", "application/json", ApiError)
+      }
+    }
+  end
+
+  def index_operation do
+    %Operation{
+      tags: ["Emoji Packs"],
+      summary: "Lists local custom emoji packs",
+      operationId: "PleromaAPI.EmojiPackController.index",
+      responses: %{
+        200 => emoji_packs_response()
+      }
+    }
+  end
+
+  def show_operation do
+    %Operation{
+      tags: ["Emoji Packs"],
+      summary: "Show emoji pack",
+      operationId: "PleromaAPI.EmojiPackController.show",
+      parameters: [name_param()],
+      responses: %{
+        200 => Operation.response("Emoji Pack", "application/json", emoji_pack()),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  def archive_operation do
+    %Operation{
+      tags: ["Emoji Packs"],
+      summary: "Requests a local pack archive from the instance",
+      operationId: "PleromaAPI.EmojiPackController.archive",
+      parameters: [name_param()],
+      responses: %{
+        200 =>
+          Operation.response("Archive file", "application/octet-stream", %Schema{
+            type: :string,
+            format: :binary
+          }),
+        403 => Operation.response("Forbidden", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  def download_operation do
+    %Operation{
+      tags: ["Emoji Packs"],
+      summary: "Download pack from another instance",
+      operationId: "PleromaAPI.EmojiPackController.download",
+      security: [%{"oAuth" => ["write"]}],
+      requestBody: request_body("Parameters", download_request(), required: true),
+      responses: %{
+        200 => ok_response(),
+        500 => Operation.response("Error", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp download_request do
+    %Schema{
+      type: :object,
+      required: [:url, :name],
+      properties: %{
+        url: %Schema{
+          type: :string,
+          format: :uri,
+          description: "URL of the instance to download from"
+        },
+        name: %Schema{type: :string, format: :uri, description: "Pack Name"},
+        as: %Schema{type: :string, format: :uri, description: "Save as"}
+      }
+    }
+  end
+
+  def create_operation do
+    %Operation{
+      tags: ["Emoji Packs"],
+      summary: "Create an empty pack",
+      operationId: "PleromaAPI.EmojiPackController.create",
+      security: [%{"oAuth" => ["write"]}],
+      parameters: [name_param()],
+      responses: %{
+        200 => ok_response(),
+        400 => Operation.response("Not Found", "application/json", ApiError),
+        409 => Operation.response("Conflict", "application/json", ApiError),
+        500 => Operation.response("Error", "application/json", ApiError)
+      }
+    }
+  end
+
+  def delete_operation do
+    %Operation{
+      tags: ["Emoji Packs"],
+      summary: "Delete a custom emoji pack",
+      operationId: "PleromaAPI.EmojiPackController.delete",
+      security: [%{"oAuth" => ["write"]}],
+      parameters: [name_param()],
+      responses: %{
+        200 => ok_response(),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  def update_operation do
+    %Operation{
+      tags: ["Emoji Packs"],
+      summary: "Updates (replaces) pack metadata",
+      operationId: "PleromaAPI.EmojiPackController.update",
+      security: [%{"oAuth" => ["write"]}],
+      requestBody: request_body("Parameters", update_request(), required: true),
+      parameters: [name_param()],
+      responses: %{
+        200 => Operation.response("Metadata", "application/json", metadata()),
+        400 => Operation.response("Bad Request", "application/json", ApiError)
+      }
+    }
+  end
+
+  def add_file_operation do
+    %Operation{
+      tags: ["Emoji Packs"],
+      summary: "Add new file to the pack",
+      operationId: "PleromaAPI.EmojiPackController.add_file",
+      security: [%{"oAuth" => ["write"]}],
+      requestBody: request_body("Parameters", add_file_request(), required: true),
+      parameters: [name_param()],
+      responses: %{
+        200 => Operation.response("Files Object", "application/json", files_object()),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        409 => Operation.response("Conflict", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp add_file_request do
+    %Schema{
+      type: :object,
+      required: [:file],
+      properties: %{
+        file: %Schema{
+          description:
+            "File needs to be uploaded with the multipart request or link to remote file",
+          anyOf: [
+            %Schema{type: :string, format: :binary},
+            %Schema{type: :string, format: :uri}
+          ]
+        },
+        shortcode: %Schema{
+          type: :string,
+          description:
+            "Shortcode for new emoji, must be unique for all emoji. If not sended, shortcode will be taken from original filename."
+        },
+        filename: %Schema{
+          type: :string,
+          description:
+            "New emoji file name. If not specified will be taken from original filename."
+        }
+      }
+    }
+  end
+
+  def update_file_operation do
+    %Operation{
+      tags: ["Emoji Packs"],
+      summary: "Add new file to the pack",
+      operationId: "PleromaAPI.EmojiPackController.update_file",
+      security: [%{"oAuth" => ["write"]}],
+      requestBody: request_body("Parameters", update_file_request(), required: true),
+      parameters: [name_param()],
+      responses: %{
+        200 => Operation.response("Files Object", "application/json", files_object()),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        409 => Operation.response("Conflict", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp update_file_request do
+    %Schema{
+      type: :object,
+      required: [:shortcode, :new_shortcode, :new_filename],
+      properties: %{
+        shortcode: %Schema{
+          type: :string,
+          description: "Emoji file shortcode"
+        },
+        new_shortcode: %Schema{
+          type: :string,
+          description: "New emoji file shortcode"
+        },
+        new_filename: %Schema{
+          type: :string,
+          description: "New filename for emoji file"
+        },
+        force: %Schema{
+          type: :boolean,
+          description: "With true value to overwrite existing emoji with new shortcode",
+          default: false
+        }
+      }
+    }
+  end
+
+  def delete_file_operation do
+    %Operation{
+      tags: ["Emoji Packs"],
+      summary: "Delete emoji file from pack",
+      operationId: "PleromaAPI.EmojiPackController.delete_file",
+      security: [%{"oAuth" => ["write"]}],
+      parameters: [
+        name_param(),
+        Operation.parameter(:shortcode, :query, :string, "File shortcode",
+          example: "cofe",
+          required: true
+        )
+      ],
+      responses: %{
+        200 => Operation.response("Files Object", "application/json", files_object()),
+        400 => Operation.response("Bad Request", "application/json", ApiError)
+      }
+    }
+  end
+
+  def import_from_filesystem_operation do
+    %Operation{
+      tags: ["Emoji Packs"],
+      summary: "Imports packs from filesystem",
+      operationId: "PleromaAPI.EmojiPackController.import",
+      security: [%{"oAuth" => ["write"]}],
+      responses: %{
+        200 =>
+          Operation.response("Array of imported pack names", "application/json", %Schema{
+            type: :array,
+            items: %Schema{type: :string}
+          })
+      }
+    }
+  end
+
+  defp name_param do
+    Operation.parameter(:name, :path, :string, "Pack Name", example: "cofe", required: true)
+  end
+
+  defp url_param do
+    Operation.parameter(
+      :url,
+      :query,
+      %Schema{type: :string, format: :uri},
+      "URL of the instance",
+      required: true
+    )
+  end
+
+  defp ok_response do
+    Operation.response("Ok", "application/json", %Schema{type: :string, example: "ok"})
+  end
+
+  defp emoji_packs_response do
+    Operation.response(
+      "Object with pack names as keys and pack contents as values",
+      "application/json",
+      %Schema{
+        type: :object,
+        additionalProperties: emoji_pack(),
+        example: %{
+          "emojos" => emoji_pack().example
+        }
+      }
+    )
+  end
+
+  defp emoji_pack do
+    %Schema{
+      title: "EmojiPack",
+      type: :object,
+      properties: %{
+        files: files_object(),
+        pack: %Schema{
+          type: :object,
+          properties: %{
+            license: %Schema{type: :string},
+            homepage: %Schema{type: :string, format: :uri},
+            description: %Schema{type: :string},
+            "can-download": %Schema{type: :boolean},
+            "share-files": %Schema{type: :boolean},
+            "download-sha256": %Schema{type: :string}
+          }
+        }
+      },
+      example: %{
+        "files" => %{"emacs" => "emacs.png", "guix" => "guix.png"},
+        "pack" => %{
+          "license" => "Test license",
+          "homepage" => "https://pleroma.social",
+          "description" => "Test description",
+          "can-download" => true,
+          "share-files" => true,
+          "download-sha256" => "57482F30674FD3DE821FF48C81C00DA4D4AF1F300209253684ABA7075E5FC238"
+        }
+      }
+    }
+  end
+
+  defp files_object do
+    %Schema{
+      type: :object,
+      additionalProperties: %Schema{type: :string},
+      description: "Object with emoji names as keys and filenames as values"
+    }
+  end
+
+  defp update_request do
+    %Schema{
+      type: :object,
+      properties: %{
+        metadata: %Schema{
+          type: :object,
+          description: "Metadata to replace the old one",
+          properties: %{
+            license: %Schema{type: :string},
+            homepage: %Schema{type: :string, format: :uri},
+            description: %Schema{type: :string},
+            "fallback-src": %Schema{
+              type: :string,
+              format: :uri,
+              description: "Fallback url to download pack from"
+            },
+            "fallback-src-sha256": %Schema{
+              type: :string,
+              description: "SHA256 encoded for fallback pack archive"
+            },
+            "share-files": %Schema{type: :boolean, description: "Is pack allowed for sharing?"}
+          }
+        }
+      }
+    }
+  end
+
+  defp metadata do
+    %Schema{
+      type: :object,
+      properties: %{
+        license: %Schema{type: :string},
+        homepage: %Schema{type: :string, format: :uri},
+        description: %Schema{type: :string},
+        "fallback-src": %Schema{
+          type: :string,
+          format: :uri,
+          description: "Fallback url to download pack from"
+        },
+        "fallback-src-sha256": %Schema{
+          type: :string,
+          description: "SHA256 encoded for fallback pack archive"
+        },
+        "share-files": %Schema{type: :boolean, description: "Is pack allowed for sharing?"}
+      }
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/pleroma_mascot_operation.ex

@@ -0,0 +1,79 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.PleromaMascotOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def show_operation do
+    %Operation{
+      tags: ["Mascot"],
+      summary: "Gets user mascot image",
+      security: [%{"oAuth" => ["read:accounts"]}],
+      operationId: "PleromaAPI.MascotController.show",
+      responses: %{
+        200 => Operation.response("Mascot", "application/json", mascot())
+      }
+    }
+  end
+
+  def update_operation do
+    %Operation{
+      tags: ["Mascot"],
+      summary: "Set/clear user avatar image",
+      description:
+        "Behaves exactly the same as `POST /api/v1/upload`. Can only accept images - any attempt to upload non-image files will be met with `HTTP 415 Unsupported Media Type`.",
+      operationId: "PleromaAPI.MascotController.update",
+      requestBody:
+        request_body(
+          "Parameters",
+          %Schema{
+            type: :object,
+            properties: %{
+              file: %Schema{type: :string, format: :binary}
+            }
+          },
+          required: true
+        ),
+      security: [%{"oAuth" => ["write:accounts"]}],
+      responses: %{
+        200 => Operation.response("Mascot", "application/json", mascot()),
+        415 => Operation.response("Unsupported Media Type", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp mascot do
+    %Schema{
+      type: :object,
+      properties: %{
+        id: %Schema{type: :string},
+        url: %Schema{type: :string, format: :uri},
+        type: %Schema{type: :string},
+        pleroma: %Schema{
+          type: :object,
+          properties: %{
+            mime_type: %Schema{type: :string}
+          }
+        }
+      },
+      example: %{
+        "id" => "abcdefg",
+        "url" => "https://pleroma.example.org/media/abcdefg.png",
+        "type" => "image",
+        "pleroma" => %{
+          "mime_type" => "image/png"
+        }
+      }
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex

@@ -0,0 +1,48 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.PleromaNotificationOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.NotificationOperation
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def mark_as_read_operation do
+    %Operation{
+      tags: ["Notifications"],
+      summary: "Mark notifications as read. Query parameters are mutually exclusive.",
+      requestBody:
+        request_body("Parameters", %Schema{
+          type: :object,
+          properties: %{
+            id: %Schema{type: :integer, description: "A single notification ID to read"},
+            max_id: %Schema{type: :integer, description: "Read all notifications up to this ID"}
+          }
+        }),
+      security: [%{"oAuth" => ["write:notifications"]}],
+      operationId: "PleromaAPI.NotificationController.mark_as_read",
+      responses: %{
+        200 =>
+          Operation.response(
+            "A Notification or array of Motifications",
+            "application/json",
+            %Schema{
+              anyOf: [
+                %Schema{type: :array, items: NotificationOperation.notification()},
+                NotificationOperation.notification()
+              ]
+            }
+          ),
+        400 => Operation.response("Bad Request", "application/json", ApiError)
+      }
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex

@@ -0,0 +1,102 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Reference
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.Account
+  alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def create_operation do
+    %Operation{
+      tags: ["Scrobbles"],
+      summary: "Creates a new Listen activity for an account",
+      security: [%{"oAuth" => ["write"]}],
+      operationId: "PleromaAPI.ScrobbleController.create",
+      requestBody: request_body("Parameters", create_request(), requried: true),
+      responses: %{
+        200 => Operation.response("Scrobble", "application/json", scrobble())
+      }
+    }
+  end
+
+  def index_operation do
+    %Operation{
+      tags: ["Scrobbles"],
+      summary: "Requests a list of current and recent Listen activities for an account",
+      operationId: "PleromaAPI.ScrobbleController.index",
+      parameters: [
+        %Reference{"$ref": "#/components/parameters/accountIdOrNickname"} | pagination_params()
+      ],
+      security: [%{"oAuth" => ["read"]}],
+      responses: %{
+        200 =>
+          Operation.response("Array of Scrobble", "application/json", %Schema{
+            type: :array,
+            items: scrobble()
+          })
+      }
+    }
+  end
+
+  defp create_request do
+    %Schema{
+      type: :object,
+      required: [:title],
+      properties: %{
+        title: %Schema{type: :string, description: "The title of the media playing"},
+        album: %Schema{type: :string, description: "The album of the media playing"},
+        artist: %Schema{type: :string, description: "The artist of the media playing"},
+        length: %Schema{type: :integer, description: "The length of the media playing"},
+        visibility: %Schema{
+          allOf: [VisibilityScope],
+          default: "public",
+          description: "Scrobble visibility"
+        }
+      },
+      example: %{
+        "title" => "Some Title",
+        "artist" => "Some Artist",
+        "album" => "Some Album",
+        "length" => 180_000
+      }
+    }
+  end
+
+  defp scrobble do
+    %Schema{
+      type: :object,
+      properties: %{
+        id: %Schema{type: :string},
+        account: Account,
+        title: %Schema{type: :string, description: "The title of the media playing"},
+        album: %Schema{type: :string, description: "The album of the media playing"},
+        artist: %Schema{type: :string, description: "The artist of the media playing"},
+        length: %Schema{
+          type: :integer,
+          description: "The length of the media playing",
+          nullable: true
+        },
+        created_at: %Schema{type: :string, format: :"date-time"}
+      },
+      example: %{
+        "id" => "1234",
+        "account" => Account.schema().example,
+        "title" => "Some Title",
+        "artist" => "Some Artist",
+        "album" => "Some Album",
+        "length" => 180_000,
+        "created_at" => "2019-09-28T12:40:45.000Z"
+      }
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/report_operation.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ApiSpec.ReportOperation do
   alias OpenApiSpex.Schema
   alias Pleroma.Web.ApiSpec.Helpers
   alias Pleroma.Web.ApiSpec.Schemas.ApiError
+  alias Pleroma.Web.ApiSpec.Schemas.BooleanLike
 
   def open_api_operation(action) do
     operation = String.to_existing_atom("#{action}_operation")
@@ -47,7 +48,7 @@ defmodule Pleroma.Web.ApiSpec.ReportOperation do
           description: "Reason for the report"
         },
         forward: %Schema{
-          type: :boolean,
+          allOf: [BooleanLike],
           nullable: true,
           default: false,
           description:

lib/pleroma/web/api_spec/operations/status_operation.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do
   alias OpenApiSpex.Schema
   alias Pleroma.Web.ApiSpec.AccountOperation
   alias Pleroma.Web.ApiSpec.Schemas.ApiError
+  alias Pleroma.Web.ApiSpec.Schemas.BooleanLike
   alias Pleroma.Web.ApiSpec.Schemas.FlakeID
   alias Pleroma.Web.ApiSpec.Schemas.ScheduledStatus
   alias Pleroma.Web.ApiSpec.Schemas.Status
@@ -394,12 +395,12 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do
                 "Duration the poll should be open, in seconds. Must be provided with `poll[options]`"
             },
             multiple: %Schema{
-              type: :boolean,
+              allOf: [BooleanLike],
               nullable: true,
               description: "Allow multiple choices?"
             },
             hide_totals: %Schema{
-              type: :boolean,
+              allOf: [BooleanLike],
               nullable: true,
               description: "Hide vote counts until the poll ends?"
             }
@@ -411,7 +412,7 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do
           description: "ID of the status being replied to, if status is a reply"
         },
         sensitive: %Schema{
-          type: :boolean,
+          allOf: [BooleanLike],
           nullable: true,
           description: "Mark status and attached media as sensitive?"
         },
@@ -435,7 +436,7 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do
         },
         # Pleroma-specific properties:
         preview: %Schema{
-          type: :boolean,
+          allOf: [BooleanLike],
           nullable: true,
           description:
             "If set to `true` the post won't be actually posted, but the status entitiy would still be rendered back. This could be useful for previewing rich text/custom emoji, for example"
@@ -486,7 +487,7 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do
     }
   end
 
-  defp id_param do
+  def id_param do
     Operation.parameter(:id, :path, FlakeID, "Status ID",
       example: "9umDrYheeY451cQnEe",
       required: true

lib/pleroma/web/api_spec/operations/subscription_operation.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ApiSpec.SubscriptionOperation do
   alias OpenApiSpex.Schema
   alias Pleroma.Web.ApiSpec.Helpers
   alias Pleroma.Web.ApiSpec.Schemas.ApiError
+  alias Pleroma.Web.ApiSpec.Schemas.BooleanLike
   alias Pleroma.Web.ApiSpec.Schemas.PushSubscription
 
   def open_api_operation(action) do
@@ -117,27 +118,27 @@ defmodule Pleroma.Web.ApiSpec.SubscriptionOperation do
               type: :object,
               properties: %{
                 follow: %Schema{
-                  type: :boolean,
+                  allOf: [BooleanLike],
                   nullable: true,
                   description: "Receive follow notifications?"
                 },
                 favourite: %Schema{
-                  type: :boolean,
+                  allOf: [BooleanLike],
                   nullable: true,
                   description: "Receive favourite notifications?"
                 },
                 reblog: %Schema{
-                  type: :boolean,
+                  allOf: [BooleanLike],
                   nullable: true,
                   description: "Receive reblog notifications?"
                 },
                 mention: %Schema{
-                  type: :boolean,
+                  allOf: [BooleanLike],
                   nullable: true,
                   description: "Receive mention notifications?"
                 },
                 poll: %Schema{
-                  type: :boolean,
+                  allOf: [BooleanLike],
                   nullable: true,
                   description: "Receive poll notifications?"
                 }
@@ -181,27 +182,27 @@ defmodule Pleroma.Web.ApiSpec.SubscriptionOperation do
               type: :object,
               properties: %{
                 follow: %Schema{
-                  type: :boolean,
+                  allOf: [BooleanLike],
                   nullable: true,
                   description: "Receive follow notifications?"
                 },
                 favourite: %Schema{
-                  type: :boolean,
+                  allOf: [BooleanLike],
                   nullable: true,
                   description: "Receive favourite notifications?"
                 },
                 reblog: %Schema{
-                  type: :boolean,
+                  allOf: [BooleanLike],
                   nullable: true,
                   description: "Receive reblog notifications?"
                 },
                 mention: %Schema{
-                  type: :boolean,
+                  allOf: [BooleanLike],
                   nullable: true,
                   description: "Receive mention notifications?"
                 },
                 poll: %Schema{
-                  type: :boolean,
+                  allOf: [BooleanLike],
                   nullable: true,
                   description: "Receive poll notifications?"
                 }

lib/pleroma/web/api_spec/operations/timeline_operation.ex

@@ -43,7 +43,7 @@ defmodule Pleroma.Web.ApiSpec.TimelineOperation do
       description:
         "View statuses with a “direct” privacy, from your account or in your notifications",
       deprecated: true,
-      parameters: pagination_params(),
+      parameters: [with_muted_param() | pagination_params()],
       security: [%{"oAuth" => ["read:statuses"]}],
       operationId: "TimelineController.direct",
       responses: %{

lib/pleroma/web/auth/totp_authenticator.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.Auth.TOTPAuthenticator do

lib/pleroma/web/common_api/common_api.ex

@@ -127,18 +127,19 @@ defmodule Pleroma.Web.CommonAPI do
   end
 
   def repeat(id, user, params \\ %{}) do
-    with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id(id) do
+    with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id(id),
-      object = Object.normalize(activity)
+         object = %Object{} <- Object.normalize(activity, false),
-      announce_activity = Utils.get_existing_announce(user.ap_id, object)
+         {_, nil} <- {:existing_announce, Utils.get_existing_announce(user.ap_id, object)},
-      public = public_announce?(object, params)
+         public = public_announce?(object, params),
-
+         {:ok, announce, _} <- Builder.announce(user, object, public: public),
-      if announce_activity do
+         {:ok, activity, _} <- Pipeline.common_pipeline(announce, local: true) do
-        {:ok, announce_activity, object}
+      {:ok, activity}
-      else
-        ActivityPub.announce(user, object, nil, true, public)
-      end
     else
-      _ -> {:error, :not_found}
+      {:existing_announce, %Activity{} = announce} ->
+        {:ok, announce}
+
+      _ ->
+        {:error, :not_found}
     end
   end
 
@@ -347,11 +348,14 @@ defmodule Pleroma.Web.CommonAPI do
     |> check_expiry_date()
   end
 
-  def listen(user, %{"title" => _} = data) do
+  def listen(user, data) do
-    with visibility <- data["visibility"] || "public",
+    visibility = Map.get(data, :visibility, "public")
-         {to, cc} <- get_to_and_cc(user, [], nil, visibility, nil),
+
+    with {to, cc} <- get_to_and_cc(user, [], nil, visibility, nil),
          listen_data <-
-           Map.take(data, ["album", "artist", "title", "length"])
+           data
+           |> Map.take([:album, :artist, :title, :length])
+           |> Map.new(fn {key, value} -> {to_string(key), value} end)
            |> Map.put("type", "Audio")
            |> Map.put("to", to)
            |> Map.put("cc", cc)

lib/pleroma/web/common_api/utils.ex

@@ -102,7 +102,8 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   end
 
   def get_to_and_cc(_user, mentioned_users, inReplyTo, "direct", _) do
-    if inReplyTo do
+    # If the OP is a DM already, add the implicit actor.
+    if inReplyTo && Visibility.is_direct?(inReplyTo) do
       {Enum.uniq([inReplyTo.data["actor"] | mentioned_users]), []}
     else
       {mentioned_users, []}
@@ -395,10 +396,12 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   def to_masto_date(_), do: ""
 
   defp shortname(name) do
-    if String.length(name) < 30 do
+    with max_length when max_length > 0 <-
-      name
+           Config.get([Pleroma.Upload, :filename_display_max_length], 30),
+         true <- String.length(name) > max_length do
+      String.slice(name, 0..max_length) <> "…"
     else
-      String.slice(name, 0..30) <> "…"
+      _ -> name
     end
   end
 
@@ -467,6 +470,8 @@ defmodule Pleroma.Web.CommonAPI.Utils do
         |> Enum.map(& &1.ap_id)
 
       recipients ++ subscriber_ids
+    else
+      _e -> recipients
     end
   end
 
@@ -478,6 +483,8 @@ defmodule Pleroma.Web.CommonAPI.Utils do
       |> User.get_followers()
       |> Enum.map(& &1.ap_id)
       |> Enum.concat(recipients)
+    else
+      _e -> recipients
     end
   end
 

lib/pleroma/web/controller_helper.ex

@@ -5,6 +5,8 @@
 defmodule Pleroma.Web.ControllerHelper do
   use Pleroma.Web, :controller
 
+  alias Pleroma.Pagination
+
   # As in Mastodon API, per https://api.rubyonrails.org/classes/ActiveModel/Type/Boolean.html
   @falsy_param_values [false, 0, "0", "f", "F", "false", "False", "FALSE", "off", "OFF"]
 
@@ -46,33 +48,8 @@ defmodule Pleroma.Web.ControllerHelper do
     do: conn
 
   def add_link_headers(conn, activities, extra_params) do
-    case List.last(activities) do
+    case get_pagination_fields(conn, activities, extra_params) do
-      %{id: max_id} ->
+      %{"next" => next_url, "prev" => prev_url} ->
-        params =
-          conn.params
-          |> Map.drop(Map.keys(conn.path_params))
-          |> Map.drop(["since_id", "max_id", "min_id"])
-          |> Map.merge(extra_params)
-
-        limit =
-          params
-          |> Map.get("limit", "20")
-          |> String.to_integer()
-
-        min_id =
-          if length(activities) <= limit do
-            activities
-            |> List.first()
-            |> Map.get(:id)
-          else
-            activities
-            |> Enum.at(limit * -1)
-            |> Map.get(:id)
-          end
-
-        next_url = current_url(conn, Map.merge(params, %{max_id: max_id}))
-        prev_url = current_url(conn, Map.merge(params, %{min_id: min_id}))
-
         put_resp_header(conn, "link", "<#{next_url}>; rel=\"next\", <#{prev_url}>; rel=\"prev\"")
 
       _ ->
@@ -80,9 +57,43 @@ defmodule Pleroma.Web.ControllerHelper do
     end
   end
 
+  def get_pagination_fields(conn, activities, extra_params \\ %{}) do
+    case List.last(activities) do
+      %{id: max_id} ->
+        params =
+          conn.params
+          |> Map.drop(Map.keys(conn.path_params))
+          |> Map.merge(extra_params)
+          |> Map.drop(Pagination.page_keys() -- ["limit", "order"])
+
+        min_id =
+          activities
+          |> List.first()
+          |> Map.get(:id)
+
+        fields = %{
+          "next" => current_url(conn, Map.put(params, :max_id, max_id)),
+          "prev" => current_url(conn, Map.put(params, :min_id, min_id))
+        }
+
+        #  Generating an `id` without already present pagination keys would
+        # need a query-restriction with an `q.id >= ^id` or `q.id <= ^id`
+        # instead of the `q.id > ^min_id` and `q.id < ^max_id`.
+        #  This is because we only have ids present inside of the page, while
+        # `min_id`, `since_id` and `max_id` requires to know one outside of it.
+        if Map.take(conn.params, Pagination.page_keys() -- ["limit", "order"]) != [] do
+          Map.put(fields, "id", current_url(conn, conn.params))
+        else
+          fields
+        end
+
+      _ ->
+        %{}
+    end
+  end
+
   def assign_account_by_id(conn, _) do
-    # TODO: use `conn.params[:id]` only after moving to OpenAPI
+    case Pleroma.User.get_cached_by_id(conn.params.id) do
-    case Pleroma.User.get_cached_by_id(conn.params[:id] || conn.params["id"]) do
       %Pleroma.User{} = account -> assign(conn, :account, account)
       nil -> Pleroma.Web.MastodonAPI.FallbackController.call(conn, {:error, :not_found}) |> halt()
     end
@@ -99,11 +110,6 @@ defmodule Pleroma.Web.ControllerHelper do
     render_error(conn, :not_implemented, "Can't display this activity")
   end
 
-  @spec put_if_exist(map(), atom() | String.t(), any) :: map()
-  def put_if_exist(map, _key, nil), do: map
-
-  def put_if_exist(map, key, value), do: Map.put(map, key, value)
-
   @doc """
   Returns true if request specifies to include embedded relationships in account objects.
   May only be used in selected account-related endpoints; has no effect for status- or

lib/pleroma/web/embed_controller.ex

@@ -0,0 +1,42 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.EmbedController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Activity
+  alias Pleroma.Object
+  alias Pleroma.User
+
+  alias Pleroma.Web.ActivityPub.Visibility
+
+  plug(:put_layout, :embed)
+
+  def show(conn, %{"id" => id}) do
+    with %Activity{local: true} = activity <-
+           Activity.get_by_id_with_object(id),
+         true <- Visibility.is_public?(activity.object) do
+      {:ok, author} = User.get_or_fetch(activity.object.data["actor"])
+
+      conn
+      |> delete_resp_header("x-frame-options")
+      |> delete_resp_header("content-security-policy")
+      |> render("show.html",
+        activity: activity,
+        author: User.sanitize_html(author),
+        counts: get_counts(activity)
+      )
+    end
+  end
+
+  defp get_counts(%Activity{} = activity) do
+    %Object{data: data} = Object.normalize(activity)
+
+    %{
+      likes: Map.get(data, "like_count", 0),
+      replies: Map.get(data, "repliesCount", 0),
+      announces: Map.get(data, "announcement_count", 0)
+    }
+  end
+end

lib/pleroma/web/feed/tag_controller.ex

@@ -9,14 +9,12 @@ defmodule Pleroma.Web.Feed.TagController do
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.Feed.FeedView
 
-  import Pleroma.Web.ControllerHelper, only: [put_if_exist: 3]
-
   def feed(conn, %{"tag" => raw_tag} = params) do
     {format, tag} = parse_tag(raw_tag)
 
     activities =
-      %{"type" => ["Create"], "tag" => tag}
+      %{type: ["Create"], tag: tag}
-      |> put_if_exist("max_id", params["max_id"])
+      |> Pleroma.Maps.put_if_present(:max_id, params["max_id"])
       |> ActivityPub.fetch_public_activities()
 
     conn

lib/pleroma/web/feed/user_controller.ex

@@ -11,8 +11,6 @@ defmodule Pleroma.Web.Feed.UserController do
   alias Pleroma.Web.ActivityPub.ActivityPubController
   alias Pleroma.Web.Feed.FeedView
 
-  import Pleroma.Web.ControllerHelper, only: [put_if_exist: 3]
-
   plug(Pleroma.Plugs.SetFormatPlug when action in [:feed_redirect])
 
   action_fallback(:errors)
@@ -52,11 +50,11 @@ defmodule Pleroma.Web.Feed.UserController do
     with {_, %User{} = user} <- {:fetch_user, User.get_cached_by_nickname(nickname)} do
       activities =
         %{
-          "type" => ["Create"],
+          type: ["Create"],
-          "actor_id" => user.ap_id
+          actor_id: user.ap_id
         }
-        |> put_if_exist("max_id", params["max_id"])
+        |> Pleroma.Maps.put_if_present(:max_id, params["max_id"])
-        |> ActivityPub.fetch_public_activities()
+        |> ActivityPub.fetch_public_or_unlisted_activities()
 
       conn
       |> put_resp_content_type("application/#{format}+xml")

lib/pleroma/web/mastodon_api/controllers/account_controller.ex

@@ -14,6 +14,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
       json_response: 3
     ]
 
+  alias Pleroma.Maps
   alias Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug
   alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.Plugs.RateLimiter
@@ -81,7 +82,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
 
   plug(
     RateLimiter,
-    [name: :relation_id_action, params: ["id", "uri"]] when action in @relationship_actions
+    [name: :relation_id_action, params: [:id, :uri]] when action in @relationship_actions
   )
 
   plug(RateLimiter, [name: :relations_actions] when action in @relationship_actions)
@@ -139,9 +140,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
   end
 
   @doc "PATCH /api/v1/accounts/update_credentials"
-  def update_credentials(%{assigns: %{user: original_user}, body_params: params} = conn, _params) do
+  def update_credentials(%{assigns: %{user: user}, body_params: params} = conn, _params) do
-    user = original_user
-
     params =
       params
       |> Enum.filter(fn {_, value} -> not is_nil(value) end)
@@ -162,39 +161,49 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
         :discoverable
       ]
       |> Enum.reduce(%{}, fn key, acc ->
-        add_if_present(acc, params, key, key, &{:ok, truthy_param?(&1)})
+        Maps.put_if_present(acc, key, params[key], &{:ok, truthy_param?(&1)})
       end)
-      |> add_if_present(params, :display_name, :name)
+      |> Maps.put_if_present(:name, params[:display_name])
-      |> add_if_present(params, :note, :bio)
+      |> Maps.put_if_present(:bio, params[:note])
-      |> add_if_present(params, :avatar, :avatar)
+      |> Maps.put_if_present(:avatar, params[:avatar])
-      |> add_if_present(params, :header, :banner)
+      |> Maps.put_if_present(:banner, params[:header])
-      |> add_if_present(params, :pleroma_background_image, :background)
+      |> Maps.put_if_present(:background, params[:pleroma_background_image])
-      |> add_if_present(
+      |> Maps.put_if_present(
-        params,
-        :fields_attributes,
         :raw_fields,
+        params[:fields_attributes],
         &{:ok, normalize_fields_attributes(&1)}
       )
-      |> add_if_present(params, :pleroma_settings_store, :pleroma_settings_store)
+      |> Maps.put_if_present(:pleroma_settings_store, params[:pleroma_settings_store])
-      |> add_if_present(params, :default_scope, :default_scope)
+      |> Maps.put_if_present(:default_scope, params[:default_scope])
-      |> add_if_present(params, :actor_type, :actor_type)
+      |> Maps.put_if_present(:default_scope, params["source"]["privacy"])
+      |> Maps.put_if_present(:actor_type, params[:actor_type])
 
     changeset = User.update_changeset(user, user_params)
 
     with {:ok, user} <- User.update_and_set_cache(changeset) do
+      user
+      |> build_update_activity_params()
+      |> ActivityPub.update()
+
       render(conn, "show.json", user: user, for: user, with_pleroma_settings: true)
     else
       _e -> render_error(conn, :forbidden, "Invalid request")
     end
   end
 
-  defp add_if_present(map, params, params_field, map_field, value_function \\ &{:ok, &1}) do
+  # Hotfix, handling will be redone with the pipeline
-    with true <- Map.has_key?(params, params_field),
+  defp build_update_activity_params(user) do
-         {:ok, new_value} <- value_function.(Map.get(params, params_field)) do
+    object =
-      Map.put(map, map_field, new_value)
+      Pleroma.Web.ActivityPub.UserView.render("user.json", user: user)
-    else
+      |> Map.delete("@context")
-      _ -> map
+
-    end
+    %{
+      local: true,
+      to: [user.follower_address],
+      cc: [],
+      object: object,
+      actor: user.ap_id
+    }
   end
 
   defp normalize_fields_attributes(fields) do
@@ -235,9 +244,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
       params =
         params
         |> Map.delete(:tagged)
-        |> Enum.filter(&(not is_nil(&1)))
+        |> Map.put(:tag, params[:tagged])
-        |> Map.new(fn {key, value} -> {to_string(key), value} end)
-        |> Map.put("tag", params[:tagged])
 
       activities = ActivityPub.fetch_user_activities(user, reading_user, params)